From 75d2f4d2827261c3ae6277aabe73cfed943aedde Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 9 Nov 2024 22:52:50 +0900 Subject: [PATCH 1/3] Add Spotless --- build.gradle.kts | 11 + .../aws/s3/FluencyBuilderForAwsS3.java | 598 ++++--- .../aws/s3/ingester/AwsS3Ingester.java | 81 +- .../ingester/DefaultS3DestinationDecider.java | 156 +- .../aws/s3/ingester/S3DestinationDecider.java | 35 +- .../aws/s3/ingester/sender/AwsS3Sender.java | 535 +++---- .../s3/recordformat/AwsS3RecordFormatter.java | 5 +- .../s3/recordformat/CsvRecordFormatter.java | 300 ++-- .../s3/recordformat/JsonlRecordFormatter.java | 136 +- .../MessagePackRecordFormatter.java | 26 +- .../aws/s3/FluencyBuilderForAwsS3Test.java | 389 ++--- .../aws/s3/ingester/AwsS3IngesterTest.java | 77 +- .../DefaultS3DestinationDeciderTest.java | 89 +- .../s3/ingester/sender/AwsS3SenderTest.java | 283 ++-- .../recordformat/CsvRecordFormatterTest.java | 321 ++-- .../JsonlRecordFormatterTest.java | 198 ++- .../fluency/BufferFullException.java | 11 +- .../java/org/komamitsu/fluency/EventTime.java | 256 ++- .../java/org/komamitsu/fluency/Fluency.java | 326 ++-- .../org/komamitsu/fluency/FluencyBuilder.java | 372 +++-- .../fluency/NonRetryableException.java | 18 +- .../komamitsu/fluency/RetryableException.java | 18 +- .../org/komamitsu/fluency/buffer/Buffer.java | 1055 ++++++------- .../komamitsu/fluency/buffer/BufferPool.java | 206 ++- .../komamitsu/fluency/buffer/FileBackup.java | 352 ++--- .../komamitsu/fluency/flusher/Flusher.java | 448 +++--- .../komamitsu/fluency/ingester/Ingester.java | 12 +- .../fluency/ingester/sender/ErrorHandler.java | 5 +- .../fluency/ingester/sender/Sender.java | 33 +- .../recordformat/AbstractRecordFormatter.java | 68 +- .../MessagePackRecordFormatter.java | 122 +- .../fluency/recordformat/RecordFormatter.java | 31 +- .../recordaccessor/MapRecordAccessor.java | 76 +- .../MessagePackRecordAccessor.java | 143 +- .../recordaccessor/RecordAccessor.java | 12 +- .../fluency/util/ExecutorServiceUtils.java | 102 +- .../org/komamitsu/fluency/util/Tuple.java | 30 +- .../org/komamitsu/fluency/util/Tuple3.java | 41 +- .../fluency/validation/Validatable.java | 220 ++- .../validation/annotation/DecimalMax.java | 8 +- .../validation/annotation/DecimalMin.java | 8 +- .../fluency/validation/annotation/Max.java | 8 +- .../fluency/validation/annotation/Min.java | 8 +- .../org/komamitsu/fluency/FluencyTest.java | 314 ++-- .../fluency/JsonRecordFormatter.java | 66 +- .../fluency/buffer/BufferPoolTest.java | 288 ++-- .../komamitsu/fluency/buffer/BufferTest.java | 718 +++++---- .../fluency/buffer/FileBackupTest.java | 134 +- .../fluency/flusher/FlusherTest.java | 198 ++- .../MessagePackRecordFormatterTest.java | 206 ++- .../fluency/validation/ValidatableTest.java | 195 ++- .../fluentd/FluencyExtBuilderForFluentd.java | 57 +- .../ingester/sender/UnixSocketSender.java | 170 +- .../heartbeat/UnixSocketHeartbeater.java | 83 +- .../FluencyExtBuilderForFluentdTest.java | 137 +- .../fluency/fluentd/MockUnixSocketServer.java | 530 +++---- .../ingester/sender/UnixSocketSenderTest.java | 478 +++--- .../heartbeat/UnixSocketHeartbeaterTest.java | 172 +- .../fluentd/FluencyBuilderForFluentd.java | 454 +++--- .../fluentd/ingester/FluentdIngester.java | 161 +- .../fluency/fluentd/ingester/Response.java | 32 +- .../ingester/sender/FluentdSender.java | 118 +- .../ingester/sender/InetSocketSender.java | 90 +- .../fluentd/ingester/sender/MultiSender.java | 145 +- .../ingester/sender/NetworkSender.java | 351 ++--- .../ingester/sender/RequestOption.java | 48 +- .../ingester/sender/RetryableSender.java | 202 ++- .../fluentd/ingester/sender/SSLSender.java | 199 +-- .../ingester/sender/SSLSocketBuilder.java | 77 +- .../fluentd/ingester/sender/TCPSender.java | 154 +- .../failuredetect/FailureDetectStrategy.java | 20 +- .../sender/failuredetect/FailureDetector.java | 191 +-- .../PhiAccrualFailureDetectStrategy.java | 141 +- .../sender/heartbeat/Heartbeater.java | 173 +- .../heartbeat/InetSocketHeartbeater.java | 113 +- .../sender/heartbeat/SSLHeartbeater.java | 163 +- .../sender/heartbeat/TCPHeartbeater.java | 71 +- .../sender/heartbeat/UDPHeartbeater.java | 69 +- .../sender/retry/ConstantRetryStrategy.java | 82 +- .../ExponentialBackOffRetryStrategy.java | 135 +- .../ingester/sender/retry/RetryStrategy.java | 72 +- .../recordformat/FluentdRecordFormatter.java | 147 +- .../fluency/buffer/BufferForFluentdTest.java | 514 +++--- .../fluentd/AbstractFluentdServer.java | 433 +++-- .../fluentd/ConfigurableTestServer.java | 127 +- .../fluency/fluentd/EventTimeTest.java | 145 +- .../fluentd/FluencyBuilderForFluentdTest.java | 720 +++++---- .../fluency/fluentd/FluencyTest.java | 479 +++--- .../fluentd/FluencyTestWithMockServer.java | 1398 +++++++++-------- .../fluency/fluentd/MockTCPServer.java | 521 +++--- .../fluentd/MockTCPServerWithMetrics.java | 71 +- .../fluentd/SSLTestSocketFactories.java | 92 +- .../fluentd/ingester/FluentdIngesterTest.java | 197 ++- .../fluentd/ingester/sender/ConfigTest.java | 86 +- .../sender/MockMultiTCPServerWithMetrics.java | 115 +- .../ingester/sender/MockTCPSender.java | 146 +- .../ingester/sender/MultiSenderTest.java | 550 ++++--- .../ingester/sender/RetryableSenderTest.java | 141 +- .../ingester/sender/SSLSenderTest.java | 521 +++--- .../ingester/sender/SSLSocketBuilderTest.java | 109 +- .../ingester/sender/TCPSenderTest.java | 509 +++--- .../failuredetect/FailureDetectorTest.java | 126 +- .../PhiAccrualFailureDetectStrategyTest.java | 34 +- .../sender/heartbeat/HeartbeaterTest.java | 132 +- .../sender/heartbeat/SSLHeartbeaterTest.java | 237 ++- .../sender/heartbeat/TCPHeartbeaterTest.java | 143 +- .../sender/heartbeat/UDPHeartbeaterTest.java | 136 +- .../retry/ConstantRetryStrategyTest.java | 52 +- .../ExponentialBackOffRetryStrategyTest.java | 80 +- .../fluency/integration/WithRealFluentd.java | 370 +++-- .../FluencyBuilderForTreasureData.java | 220 ++- .../ingester/TreasureDataIngester.java | 49 +- .../ingester/sender/TreasureDataSender.java | 660 ++++---- .../FluencyBuilderForTreasureDataTest.java | 305 ++-- .../ingester/TreasureDataIngesterTest.java | 82 +- .../sender/TreasureDataSenderTest.java | 343 ++-- gradle/wrapper/gradle-wrapper.jar | Bin 59203 -> 63721 bytes gradle/wrapper/gradle-wrapper.properties | 4 +- gradlew | 282 ++-- gradlew.bat | 15 +- 120 files changed, 11756 insertions(+), 12771 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 562d7fd9..3c9b0879 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -14,6 +14,7 @@ plugins { `maven-publish` id("com.github.kt3k.coveralls") version "2.12.2" id("com.github.johnrengelman.shadow") version "8.1.1" + id("com.diffplug.spotless") version "6.25.0" } subprojects { @@ -26,6 +27,7 @@ subprojects { apply(plugin = "maven-publish") apply(plugin = "jacoco") apply(plugin = "com.github.kt3k.coveralls") + apply(plugin = "com.diffplug.spotless") repositories { mavenCentral() @@ -187,5 +189,14 @@ subprojects { tasks.coveralls { dependsOn(jacocoRootReport) } + + spotless { + java { + target("src/*/java/**/*.java") + importOrder() + removeUnusedImports() + googleJavaFormat() + } + } } diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/FluencyBuilderForAwsS3.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/FluencyBuilderForAwsS3.java index e63d875d..e1705029 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/FluencyBuilderForAwsS3.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/FluencyBuilderForAwsS3.java @@ -16,367 +16,339 @@ package org.komamitsu.fluency.aws.s3; +import java.time.ZoneId; +import java.util.List; import org.komamitsu.fluency.Fluency; +import org.komamitsu.fluency.aws.s3.ingester.AwsS3Ingester; import org.komamitsu.fluency.aws.s3.ingester.DefaultS3DestinationDecider; import org.komamitsu.fluency.aws.s3.ingester.S3DestinationDecider; +import org.komamitsu.fluency.aws.s3.ingester.sender.AwsS3Sender; import org.komamitsu.fluency.aws.s3.recordformat.AwsS3RecordFormatter; import org.komamitsu.fluency.aws.s3.recordformat.CsvRecordFormatter; import org.komamitsu.fluency.aws.s3.recordformat.JsonlRecordFormatter; import org.komamitsu.fluency.aws.s3.recordformat.MessagePackRecordFormatter; -import org.komamitsu.fluency.aws.s3.ingester.AwsS3Ingester; -import org.komamitsu.fluency.aws.s3.ingester.sender.AwsS3Sender; import org.msgpack.core.annotations.VisibleForTesting; import software.amazon.awssdk.services.s3.S3Client; -import java.time.ZoneId; -import java.util.List; - -public class FluencyBuilderForAwsS3 - extends org.komamitsu.fluency.FluencyBuilder -{ - private FormatType formatType; - private List formatCsvColumnNames; - private String awsEndpoint; - private String awsRegion; - private String awsAccessKeyId; - private String awsSecretAccessKey; - private Integer senderRetryMax; - private Integer senderRetryIntervalMillis; - private Integer senderMaxRetryIntervalMillis; - private Float senderRetryFactor; - private Integer senderWorkBufSize; - private boolean compressionEnabled = true; - private String s3KeyPrefix; - private String s3KeySuffix; - private ZoneId s3KeyTimeZoneId; - private S3DestinationDecider customS3DestinationDecider; - - public enum FormatType { - MESSAGE_PACK, - JSONL, - CSV - } - - public FluencyBuilderForAwsS3() - { - setBufferChunkRetentionTimeMillis(30 * 1000); - setBufferChunkInitialSize(4 * 1024 * 1024); - setBufferChunkRetentionSize(64 * 1024 * 1024); - } - - public FormatType getFormatType() - { - return formatType; - } - - public void setFormatType(FormatType formatType) - { - this.formatType = formatType; - } - - public List getFormatCsvColumnNames() - { - return formatCsvColumnNames; - } - - public void setFormatCsvColumnNames(List formatCsvColumnNames) - { - this.formatCsvColumnNames = formatCsvColumnNames; - } - - public String getAwsEndpoint() - { - return awsEndpoint; - } - - public void setAwsEndpoint(String awsEndpoint) - { - this.awsEndpoint = awsEndpoint; - } - - public String getAwsRegion() - { - return awsRegion; - } - - public void setAwsRegion(String awsRegion) - { - this.awsRegion = awsRegion; - } - - public String getAwsAccessKeyId() - { - return awsAccessKeyId; - } - - public void setAwsAccessKeyId(String senderAwsAccessKeyId) - { - this.awsAccessKeyId = senderAwsAccessKeyId; - } - - public String getAwsSecretAccessKey() - { - return awsSecretAccessKey; - } - - public void setAwsSecretAccessKey(String senderAwsSecretAccessKey) - { - this.awsSecretAccessKey = senderAwsSecretAccessKey; - } +public class FluencyBuilderForAwsS3 extends org.komamitsu.fluency.FluencyBuilder { + private FormatType formatType; + private List formatCsvColumnNames; + private String awsEndpoint; + private String awsRegion; + private String awsAccessKeyId; + private String awsSecretAccessKey; + private Integer senderRetryMax; + private Integer senderRetryIntervalMillis; + private Integer senderMaxRetryIntervalMillis; + private Float senderRetryFactor; + private Integer senderWorkBufSize; + private boolean compressionEnabled = true; + private String s3KeyPrefix; + private String s3KeySuffix; + private ZoneId s3KeyTimeZoneId; + private S3DestinationDecider customS3DestinationDecider; + + public enum FormatType { + MESSAGE_PACK, + JSONL, + CSV + } + + public FluencyBuilderForAwsS3() { + setBufferChunkRetentionTimeMillis(30 * 1000); + setBufferChunkInitialSize(4 * 1024 * 1024); + setBufferChunkRetentionSize(64 * 1024 * 1024); + } + + public FormatType getFormatType() { + return formatType; + } + + public void setFormatType(FormatType formatType) { + this.formatType = formatType; + } + + public List getFormatCsvColumnNames() { + return formatCsvColumnNames; + } + + public void setFormatCsvColumnNames(List formatCsvColumnNames) { + this.formatCsvColumnNames = formatCsvColumnNames; + } + + public String getAwsEndpoint() { + return awsEndpoint; + } + + public void setAwsEndpoint(String awsEndpoint) { + this.awsEndpoint = awsEndpoint; + } + + public String getAwsRegion() { + return awsRegion; + } + + public void setAwsRegion(String awsRegion) { + this.awsRegion = awsRegion; + } + + public String getAwsAccessKeyId() { + return awsAccessKeyId; + } + + public void setAwsAccessKeyId(String senderAwsAccessKeyId) { + this.awsAccessKeyId = senderAwsAccessKeyId; + } + + public String getAwsSecretAccessKey() { + return awsSecretAccessKey; + } + + public void setAwsSecretAccessKey(String senderAwsSecretAccessKey) { + this.awsSecretAccessKey = senderAwsSecretAccessKey; + } + + public Integer getSenderRetryMax() { + return senderRetryMax; + } + + public void setSenderRetryMax(Integer senderRetryMax) { + this.senderRetryMax = senderRetryMax; + } + + public Integer getSenderRetryIntervalMillis() { + return senderRetryIntervalMillis; + } + + public void setSenderRetryIntervalMillis(Integer senderRetryIntervalMillis) { + this.senderRetryIntervalMillis = senderRetryIntervalMillis; + } + + public Integer getSenderMaxRetryIntervalMillis() { + return senderMaxRetryIntervalMillis; + } + + public void setSenderMaxRetryIntervalMillis(Integer senderMaxRetryIntervalMillis) { + this.senderMaxRetryIntervalMillis = senderMaxRetryIntervalMillis; + } + + public Float getSenderRetryFactor() { + return senderRetryFactor; + } + + public void setSenderRetryFactor(Float senderRetryFactor) { + this.senderRetryFactor = senderRetryFactor; + } + + public Integer getSenderWorkBufSize() { + return senderWorkBufSize; + } + + public void setSenderWorkBufSize(Integer senderWorkBufSize) { + this.senderWorkBufSize = senderWorkBufSize; + } + + public boolean isCompressionEnabled() { + return compressionEnabled; + } + + public void setCompressionEnabled(boolean compressionEnabled) { + this.compressionEnabled = compressionEnabled; + } + + public String getS3KeyPrefix() { + return s3KeyPrefix; + } + + public void setS3KeyPrefix(String s3KeyPrefix) { + this.s3KeyPrefix = s3KeyPrefix; + } + + public String getS3KeySuffix() { + return s3KeySuffix; + } + + public void setS3KeySuffix(String s3KeySuffix) { + this.s3KeySuffix = s3KeySuffix; + } - public Integer getSenderRetryMax() - { - return senderRetryMax; - } + public ZoneId getS3KeyTimeZoneId() { + return s3KeyTimeZoneId; + } - public void setSenderRetryMax(Integer senderRetryMax) - { - this.senderRetryMax = senderRetryMax; - } + public void setS3KeyTimeZoneId(ZoneId s3KeyTimeZoneId) { + this.s3KeyTimeZoneId = s3KeyTimeZoneId; + } - public Integer getSenderRetryIntervalMillis() - { - return senderRetryIntervalMillis; - } + public S3DestinationDecider getCustomS3DestinationDecider() { + return customS3DestinationDecider; + } - public void setSenderRetryIntervalMillis(Integer senderRetryIntervalMillis) - { - this.senderRetryIntervalMillis = senderRetryIntervalMillis; - } + public void setCustomS3DestinationDecider(S3DestinationDecider customS3DestinationDecider) { + this.customS3DestinationDecider = customS3DestinationDecider; + } - public Integer getSenderMaxRetryIntervalMillis() - { - return senderMaxRetryIntervalMillis; - } + private String defaultKeyPrefix(AwsS3RecordFormatter recordFormatter) { + return "." + recordFormatter.formatName(); + } - public void setSenderMaxRetryIntervalMillis(Integer senderMaxRetryIntervalMillis) - { - this.senderMaxRetryIntervalMillis = senderMaxRetryIntervalMillis; - } + public Fluency build(AwsS3RecordFormatter recordFormatter, AwsS3Sender.Config senderConfig) { + return buildFromIngester(recordFormatter, createIngester(recordFormatter, senderConfig)); + } - public Float getSenderRetryFactor() - { - return senderRetryFactor; - } + public Fluency build(AwsS3RecordFormatter recordFormatter) { + AwsS3Sender.Config senderConfig = createSenderConfig(); - public void setSenderRetryFactor(Float senderRetryFactor) - { - this.senderRetryFactor = senderRetryFactor; - } + return build(recordFormatter, senderConfig); + } - public Integer getSenderWorkBufSize() - { - return senderWorkBufSize; - } + public Fluency build() { + return build(createRecordFormatter()); + } - public void setSenderWorkBufSize(Integer senderWorkBufSize) - { - this.senderWorkBufSize = senderWorkBufSize; - } + @VisibleForTesting + AwsS3Sender createSender(AwsS3Sender.Config senderConfig) { + return new AwsS3Sender(S3Client.builder(), senderConfig); + } - public boolean isCompressionEnabled() - { - return compressionEnabled; - } + private AwsS3Ingester createIngester( + AwsS3RecordFormatter recordFormatter, AwsS3Sender.Config senderConfig) { + S3DestinationDecider s3DestinationDecider; + if (getCustomS3DestinationDecider() == null) { + DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config(); - public void setCompressionEnabled(boolean compressionEnabled) - { - this.compressionEnabled = compressionEnabled; - } + if (getS3KeyPrefix() != null) { + config.setKeyPrefix(getS3KeyPrefix()); + } - public String getS3KeyPrefix() - { - return s3KeyPrefix; - } + if (getS3KeySuffix() != null) { + config.setKeySuffix(getS3KeySuffix()); + } else { + config.setKeySuffix( + defaultKeyPrefix(recordFormatter) + (isCompressionEnabled() ? ".gz" : "")); + } - public void setS3KeyPrefix(String s3KeyPrefix) - { - this.s3KeyPrefix = s3KeyPrefix; + if (getS3KeyTimeZoneId() != null) { + config.setZoneId(getS3KeyTimeZoneId()); + } + s3DestinationDecider = new DefaultS3DestinationDecider(config); + } else { + s3DestinationDecider = getCustomS3DestinationDecider(); } - public String getS3KeySuffix() - { - return s3KeySuffix; - } + AwsS3Sender sender = createSender(senderConfig); + return new AwsS3Ingester(sender, s3DestinationDecider); + } - public void setS3KeySuffix(String s3KeySuffix) - { - this.s3KeySuffix = s3KeySuffix; + private AwsS3RecordFormatter createRecordFormatter() { + if (formatType == null) { + throw new IllegalArgumentException("format type must be set"); } - public ZoneId getS3KeyTimeZoneId() - { - return s3KeyTimeZoneId; + AwsS3RecordFormatter recordFormatter; + switch (getFormatType()) { + case MESSAGE_PACK: + recordFormatter = new MessagePackRecordFormatter(); + break; + case JSONL: + recordFormatter = new JsonlRecordFormatter(); + break; + case CSV: + CsvRecordFormatter.Config config = new CsvRecordFormatter.Config(); + config.setColumnNames(getFormatCsvColumnNames()); + recordFormatter = new CsvRecordFormatter(config); + break; + default: + throw new IllegalArgumentException("Unexpected format type: " + getFormatType()); } - public void setS3KeyTimeZoneId(ZoneId s3KeyTimeZoneId) - { - this.s3KeyTimeZoneId = s3KeyTimeZoneId; - } + return recordFormatter; + } - public S3DestinationDecider getCustomS3DestinationDecider() - { - return customS3DestinationDecider; - } + private AwsS3Sender.Config createSenderConfig() { + AwsS3Sender.Config config = new AwsS3Sender.Config(); - public void setCustomS3DestinationDecider(S3DestinationDecider customS3DestinationDecider) - { - this.customS3DestinationDecider = customS3DestinationDecider; + if (getAwsEndpoint() != null) { + config.setEndpoint(getAwsEndpoint()); } - private String defaultKeyPrefix(AwsS3RecordFormatter recordFormatter) - { - return "." + recordFormatter.formatName(); + if (getAwsRegion() != null) { + config.setRegion(getAwsRegion()); } - public Fluency build(AwsS3RecordFormatter recordFormatter, AwsS3Sender.Config senderConfig) - { - return buildFromIngester(recordFormatter, createIngester(recordFormatter, senderConfig)); + if (getAwsAccessKeyId() != null) { + config.setAwsAccessKeyId(getAwsAccessKeyId()); } - public Fluency build(AwsS3RecordFormatter recordFormatter) - { - AwsS3Sender.Config senderConfig = createSenderConfig(); - - - return build(recordFormatter, senderConfig); + if (getAwsSecretAccessKey() != null) { + config.setAwsSecretAccessKey(getAwsSecretAccessKey()); } - public Fluency build() - { - return build(createRecordFormatter()); + if (getSenderRetryMax() != null) { + config.setRetryMax(getSenderRetryMax()); } - @VisibleForTesting - AwsS3Sender createSender(AwsS3Sender.Config senderConfig) - { - return new AwsS3Sender(S3Client.builder(), senderConfig); + if (getSenderRetryIntervalMillis() != null) { + config.setRetryIntervalMs(getSenderRetryIntervalMillis()); } - private AwsS3Ingester createIngester(AwsS3RecordFormatter recordFormatter, AwsS3Sender.Config senderConfig) - { - S3DestinationDecider s3DestinationDecider; - if (getCustomS3DestinationDecider() == null) { - DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config(); - - if (getS3KeyPrefix() != null) { - config.setKeyPrefix(getS3KeyPrefix()); - } - - if (getS3KeySuffix() != null) { - config.setKeySuffix(getS3KeySuffix()); - } - else { - config.setKeySuffix( - defaultKeyPrefix(recordFormatter) + (isCompressionEnabled() ? ".gz" : "")); - } - - if (getS3KeyTimeZoneId() != null) { - config.setZoneId(getS3KeyTimeZoneId()); - } - s3DestinationDecider = new DefaultS3DestinationDecider(config); - } - else { - s3DestinationDecider = getCustomS3DestinationDecider(); - } - - AwsS3Sender sender = createSender(senderConfig); - return new AwsS3Ingester(sender, s3DestinationDecider); + if (getSenderMaxRetryIntervalMillis() != null) { + config.setMaxRetryIntervalMs(getSenderMaxRetryIntervalMillis()); } - private AwsS3RecordFormatter createRecordFormatter() - { - if (formatType == null) { - throw new IllegalArgumentException("format type must be set"); - } - - AwsS3RecordFormatter recordFormatter; - switch (getFormatType()) { - case MESSAGE_PACK: - recordFormatter = new MessagePackRecordFormatter(); - break; - case JSONL: - recordFormatter = new JsonlRecordFormatter(); - break; - case CSV: - CsvRecordFormatter.Config config = new CsvRecordFormatter.Config(); - config.setColumnNames(getFormatCsvColumnNames()); - recordFormatter = new CsvRecordFormatter(config); - break; - default: - throw new IllegalArgumentException("Unexpected format type: " + getFormatType()); - } - - return recordFormatter; + if (getSenderRetryFactor() != null) { + config.setRetryFactor(getSenderRetryFactor()); } - private AwsS3Sender.Config createSenderConfig() - { - AwsS3Sender.Config config = new AwsS3Sender.Config(); - - if (getAwsEndpoint() != null) { - config.setEndpoint(getAwsEndpoint()); - } - - if (getAwsRegion()!= null) { - config.setRegion(getAwsRegion()); - } - - if (getAwsAccessKeyId() != null) { - config.setAwsAccessKeyId(getAwsAccessKeyId()); - } - - if (getAwsSecretAccessKey() != null) { - config.setAwsSecretAccessKey(getAwsSecretAccessKey()); - } - - if (getSenderRetryMax() != null) { - config.setRetryMax(getSenderRetryMax()); - } - - if (getSenderRetryIntervalMillis() != null) { - config.setRetryIntervalMs(getSenderRetryIntervalMillis()); - } + if (getErrorHandler() != null) { + config.setErrorHandler(getErrorHandler()); + } + + if (getSenderWorkBufSize() != null) { + config.setWorkBufSize(getSenderWorkBufSize()); + } + + config.setCompressionEnabled(isCompressionEnabled()); + + return config; + } - if (getSenderMaxRetryIntervalMillis() != null) { - config.setMaxRetryIntervalMs(getSenderMaxRetryIntervalMillis()); - } - - if (getSenderRetryFactor() != null) { - config.setRetryFactor(getSenderRetryFactor()); - } - - if (getErrorHandler() != null) { - config.setErrorHandler(getErrorHandler()); - } - - if (getSenderWorkBufSize() != null) { - config.setWorkBufSize(getSenderWorkBufSize()); - } - - config.setCompressionEnabled(isCompressionEnabled()); - - return config; - } - - @Override - public String toString() - { - return "FluencyBuilderForAwsS3{" + - "formatType=" + formatType + - ", formatCsvColumnNames=" + formatCsvColumnNames + - ", awsEndpoint='" + awsEndpoint + '\'' + - ", awsRegion='" + awsRegion + '\'' + - ", senderRetryMax=" + senderRetryMax + - ", senderRetryIntervalMillis=" + senderRetryIntervalMillis + - ", senderMaxRetryIntervalMillis=" + senderMaxRetryIntervalMillis + - ", senderRetryFactor=" + senderRetryFactor + - ", senderWorkBufSize=" + senderWorkBufSize + - ", compressionEnabled=" + compressionEnabled + - ", s3KeyPrefix='" + s3KeyPrefix + '\'' + - ", s3KeySuffix='" + s3KeySuffix + '\'' + - ", s3KeyTimeZoneId=" + s3KeyTimeZoneId + - ", customS3DestinationDecider=" + customS3DestinationDecider + - "} " + super.toString(); - } + @Override + public String toString() { + return "FluencyBuilderForAwsS3{" + + "formatType=" + + formatType + + ", formatCsvColumnNames=" + + formatCsvColumnNames + + ", awsEndpoint='" + + awsEndpoint + + '\'' + + ", awsRegion='" + + awsRegion + + '\'' + + ", senderRetryMax=" + + senderRetryMax + + ", senderRetryIntervalMillis=" + + senderRetryIntervalMillis + + ", senderMaxRetryIntervalMillis=" + + senderMaxRetryIntervalMillis + + ", senderRetryFactor=" + + senderRetryFactor + + ", senderWorkBufSize=" + + senderWorkBufSize + + ", compressionEnabled=" + + compressionEnabled + + ", s3KeyPrefix='" + + s3KeyPrefix + + '\'' + + ", s3KeySuffix='" + + s3KeySuffix + + '\'' + + ", s3KeyTimeZoneId=" + + s3KeyTimeZoneId + + ", customS3DestinationDecider=" + + customS3DestinationDecider + + "} " + + super.toString(); + } } diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/AwsS3Ingester.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/AwsS3Ingester.java index fbd011d7..101ef2cc 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/AwsS3Ingester.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/AwsS3Ingester.java @@ -16,55 +16,46 @@ package org.komamitsu.fluency.aws.s3.ingester; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.time.Instant; import org.komamitsu.fluency.aws.s3.ingester.sender.AwsS3Sender; import org.komamitsu.fluency.ingester.Ingester; import org.komamitsu.fluency.ingester.sender.Sender; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.time.Instant; - -public class AwsS3Ingester - implements Ingester -{ - private static final Logger LOG = LoggerFactory.getLogger(AwsS3Ingester.class); - private final AwsS3Sender sender; - private final S3DestinationDecider s3DestinationDecider; - - public AwsS3Ingester(AwsS3Sender sender, S3DestinationDecider s3DestinationDecider) - { - this.sender = sender; - this.s3DestinationDecider = s3DestinationDecider; - } - - @Override - public void ingest(String tag, ByteBuffer dataBuffer) - throws IOException - { - S3DestinationDecider.S3Destination s3Destination = - s3DestinationDecider.decide(tag, Instant.now()); - String bucket = s3Destination.getBucket(); - String key = s3Destination.getKey(); - - sender.send(bucket, key, dataBuffer); - } - - @Override - public Sender getSender() - { - return sender; - } - - public S3DestinationDecider getS3DestinationDecider() - { - return s3DestinationDecider; - } - - @Override - public void close() - { - sender.close(); - } +public class AwsS3Ingester implements Ingester { + private static final Logger LOG = LoggerFactory.getLogger(AwsS3Ingester.class); + private final AwsS3Sender sender; + private final S3DestinationDecider s3DestinationDecider; + + public AwsS3Ingester(AwsS3Sender sender, S3DestinationDecider s3DestinationDecider) { + this.sender = sender; + this.s3DestinationDecider = s3DestinationDecider; + } + + @Override + public void ingest(String tag, ByteBuffer dataBuffer) throws IOException { + S3DestinationDecider.S3Destination s3Destination = + s3DestinationDecider.decide(tag, Instant.now()); + String bucket = s3Destination.getBucket(); + String key = s3Destination.getKey(); + + sender.send(bucket, key, dataBuffer); + } + + @Override + public Sender getSender() { + return sender; + } + + public S3DestinationDecider getS3DestinationDecider() { + return s3DestinationDecider; + } + + @Override + public void close() { + sender.close(); + } } diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/DefaultS3DestinationDecider.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/DefaultS3DestinationDecider.java index cfe1157e..5313111f 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/DefaultS3DestinationDecider.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/DefaultS3DestinationDecider.java @@ -16,111 +16,97 @@ package org.komamitsu.fluency.aws.s3.ingester; -import org.komamitsu.fluency.validation.Validatable; - import java.time.Instant; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import org.komamitsu.fluency.validation.Validatable; + +public class DefaultS3DestinationDecider implements S3DestinationDecider { + private static final DateTimeFormatter FORMATTER = + DateTimeFormatter.ofPattern("yyyy/MM/dd/HH/mm-ss-SSSSSS"); + private final Config config; + + public DefaultS3DestinationDecider(Config config) { + config.validateValues(); + this.config = config; + } + + protected String getBucket(String tag) { + return tag; + } + + protected String getKeyBase(Instant time) { + return ZonedDateTime.ofInstant(time, getZoneId()).format(FORMATTER); + } + + @Override + public S3Destination decide(String tag, Instant time) { + String keyPrefix = getKeyPrefix() == null ? "" : getKeyPrefix() + "/"; + String keySuffix = getKeySuffix() == null ? "" : getKeySuffix(); + return new S3Destination(getBucket(tag), keyPrefix + getKeyBase(time) + keySuffix); + } -public class DefaultS3DestinationDecider - implements S3DestinationDecider -{ - private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy/MM/dd/HH/mm-ss-SSSSSS"); - private final Config config; + public String getKeyPrefix() { + return config.getKeyPrefix(); + } - public DefaultS3DestinationDecider(Config config) - { - config.validateValues(); - this.config = config; + public String getKeySuffix() { + return config.getKeySuffix(); + } + + public ZoneId getZoneId() { + return config.getZoneId(); + } + + public static class Config implements Validatable { + private String keyPrefix; + + private String keySuffix; + + private ZoneId zoneId = ZoneOffset.UTC; + + public String getKeyPrefix() { + return keyPrefix; } - protected String getBucket(String tag) - { - return tag; + public void setKeyPrefix(String keyPrefix) { + this.keyPrefix = keyPrefix; } - protected String getKeyBase(Instant time) - { - return ZonedDateTime.ofInstant(time, getZoneId()).format(FORMATTER); + public String getKeySuffix() { + return keySuffix; } - @Override - public S3Destination decide(String tag, Instant time) - { - String keyPrefix = getKeyPrefix() == null ? "" : getKeyPrefix() + "/"; - String keySuffix = getKeySuffix() == null ? "" : getKeySuffix(); - return new S3Destination(getBucket(tag), keyPrefix + getKeyBase(time) + keySuffix); + public void setKeySuffix(String keySuffix) { + this.keySuffix = keySuffix; } - public String getKeyPrefix() - { - return config.getKeyPrefix(); + public ZoneId getZoneId() { + return zoneId; } - public String getKeySuffix() - { - return config.getKeySuffix(); + public void setZoneId(ZoneId zoneId) { + this.zoneId = zoneId; } - public ZoneId getZoneId() - { - return config.getZoneId(); + void validateValues() { + validate(); } - public static class Config - implements Validatable - { - private String keyPrefix; - - private String keySuffix; - - private ZoneId zoneId = ZoneOffset.UTC; - - public String getKeyPrefix() - { - return keyPrefix; - } - - public void setKeyPrefix(String keyPrefix) - { - this.keyPrefix = keyPrefix; - } - - public String getKeySuffix() - { - return keySuffix; - } - - public void setKeySuffix(String keySuffix) - { - this.keySuffix = keySuffix; - } - - public ZoneId getZoneId() - { - return zoneId; - } - - public void setZoneId(ZoneId zoneId) - { - this.zoneId = zoneId; - } - - void validateValues() - { - validate(); - } - - @Override - public String toString() - { - return "Config{" + - "keyPrefix='" + keyPrefix + '\'' + - ", keySuffix='" + keySuffix + '\'' + - ", zoneId=" + zoneId + - '}'; - } + @Override + public String toString() { + return "Config{" + + "keyPrefix='" + + keyPrefix + + '\'' + + ", keySuffix='" + + keySuffix + + '\'' + + ", zoneId=" + + zoneId + + '}'; } + } } diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/S3DestinationDecider.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/S3DestinationDecider.java index f3bef752..c32552d6 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/S3DestinationDecider.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/S3DestinationDecider.java @@ -18,29 +18,24 @@ import java.time.Instant; -public interface S3DestinationDecider -{ - class S3Destination - { - private final String bucket; - private final String key; +public interface S3DestinationDecider { + class S3Destination { + private final String bucket; + private final String key; - public S3Destination(String bucket, String key) - { - this.bucket = bucket; - this.key = key; - } + public S3Destination(String bucket, String key) { + this.bucket = bucket; + this.key = key; + } - public String getBucket() - { - return bucket; - } + public String getBucket() { + return bucket; + } - public String getKey() - { - return key; - } + public String getKey() { + return key; } + } - S3Destination decide(String tag, Instant time); + S3Destination decide(String tag, Instant time); } diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/sender/AwsS3Sender.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/sender/AwsS3Sender.java index da279696..5ff37f99 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/sender/AwsS3Sender.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/ingester/sender/AwsS3Sender.java @@ -17,6 +17,18 @@ package org.komamitsu.fluency.aws.s3.ingester.sender; import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.time.temporal.ChronoUnit; +import java.util.zip.GZIPOutputStream; import net.jodah.failsafe.Failsafe; import net.jodah.failsafe.RetryPolicy; import org.komamitsu.fluency.NonRetryableException; @@ -37,317 +49,268 @@ import software.amazon.awssdk.services.s3.S3ClientBuilder; import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.StandardOpenOption; -import java.time.temporal.ChronoUnit; -import java.util.zip.GZIPOutputStream; +public class AwsS3Sender implements Closeable, Sender { + private static final Logger LOG = LoggerFactory.getLogger(AwsS3Sender.class); + private final Config config; + private final RetryPolicy retryPolicy; + private final S3Client client; + + public AwsS3Sender(S3ClientBuilder s3ClientBuilder) { + this(s3ClientBuilder, new Config()); + } + + public AwsS3Sender(S3ClientBuilder s3ClientBuilder, Config config) { + config.validateValues(); + this.config = config; + this.retryPolicy = + new RetryPolicy() + .handleIf( + ex -> { + if (ex == null) { + // Success. Shouldn't retry. + return false; + } + + ErrorHandler errorHandler = config.getErrorHandler(); + + if (errorHandler != null) { + errorHandler.handle(ex); + } + + if (ex instanceof InterruptedException || ex instanceof NonRetryableException) { + return false; + } + + return true; + }) + .withBackoff( + getRetryInternalMs(), getMaxRetryInternalMs(), ChronoUnit.MILLIS, getRetryFactor()) + .withMaxRetries(getRetryMax()); + + this.client = buildClient(s3ClientBuilder); + } + + @VisibleForTesting + protected S3Client buildClient(S3ClientBuilder s3ClientBuilder) { + if (config.getEndpoint() != null) { + try { + URI uri = new URI(config.getEndpoint()); + s3ClientBuilder.endpointOverride(uri); + } catch (URISyntaxException e) { + throw new NonRetryableException( + String.format("Invalid endpoint. %s", config.getEndpoint()), e); + } + } -public class AwsS3Sender - implements Closeable, Sender -{ - private static final Logger LOG = LoggerFactory.getLogger(AwsS3Sender.class); - private final Config config; - private final RetryPolicy retryPolicy; - private final S3Client client; - - public AwsS3Sender(S3ClientBuilder s3ClientBuilder) - { - this(s3ClientBuilder, new Config()); + if (config.getRegion() != null) { + s3ClientBuilder.region(Region.of(config.getRegion())); } - public AwsS3Sender(S3ClientBuilder s3ClientBuilder, Config config) - { - config.validateValues(); - this.config = config; - this.retryPolicy = - new RetryPolicy(). - handleIf(ex -> { - if (ex == null) { - // Success. Shouldn't retry. - return false; - } - - ErrorHandler errorHandler = config.getErrorHandler(); - - if (errorHandler != null) { - errorHandler.handle(ex); - } - - if (ex instanceof InterruptedException || ex instanceof NonRetryableException) { - return false; - } - - return true; - }). - withBackoff( - getRetryInternalMs(), - getMaxRetryInternalMs(), - ChronoUnit.MILLIS, - getRetryFactor()). - withMaxRetries(getRetryMax()); - - this.client = buildClient(s3ClientBuilder); + if (config.getAwsAccessKeyId() != null && config.getAwsSecretAccessKey() != null) { + AwsBasicCredentials credentials = + AwsBasicCredentials.create(config.getAwsAccessKeyId(), config.getAwsSecretAccessKey()); + s3ClientBuilder.credentialsProvider(StaticCredentialsProvider.create(credentials)); } - @VisibleForTesting - protected S3Client buildClient(S3ClientBuilder s3ClientBuilder) - { - if (config.getEndpoint() != null) { - try { - URI uri = new URI(config.getEndpoint()); - s3ClientBuilder.endpointOverride(uri); - } - catch (URISyntaxException e) { - throw new NonRetryableException( - String.format("Invalid endpoint. %s", config.getEndpoint()), e); - } - } - - if (config.getRegion() != null) { - s3ClientBuilder.region(Region.of(config.getRegion())); - } - - if (config.getAwsAccessKeyId() != null && config.getAwsSecretAccessKey() != null) { - AwsBasicCredentials credentials = - AwsBasicCredentials.create(config.getAwsAccessKeyId(), config.getAwsSecretAccessKey()); - s3ClientBuilder.credentialsProvider(StaticCredentialsProvider.create(credentials)); - } - - return s3ClientBuilder.build(); + return s3ClientBuilder.build(); + } + + public int getRetryInternalMs() { + return config.getRetryIntervalMs(); + } + + public int getMaxRetryInternalMs() { + return config.getMaxRetryIntervalMs(); + } + + public float getRetryFactor() { + return config.getRetryFactor(); + } + + public int getRetryMax() { + return config.getRetryMax(); + } + + public int getWorkBufSize() { + return config.getWorkBufSize(); + } + + private void copyStreams(InputStream in, OutputStream out) throws IOException { + byte[] buf = new byte[getWorkBufSize()]; + while (true) { + int readLen = in.read(buf); + if (readLen < 0) { + break; + } + out.write(buf, 0, readLen); + } + } + + private void uploadData(String bucket, String key, File file) { + LOG.debug("Upload data to S3: bucket={}, key={}, fileSize={}", bucket, key, file.length()); + + try { + PutObjectRequest.Builder builder = PutObjectRequest.builder().bucket(bucket).key(key); + client.putObject(builder.build(), RequestBody.fromFile(file)); + } catch (NonRetryableException e) { + throw e; + } catch (Throwable e) { + throw new RetryableException( + String.format("Failed to upload data. bucket=%s, key=%s", bucket, key), e); + } + } + + public void send(String bucket, String key, ByteBuffer dataBuffer) throws IOException { + File file = File.createTempFile("tmp-fluency-", ".tmp"); + try { + try (InputStream in = new ByteBufferBackedInputStream(dataBuffer); + OutputStream fout = Files.newOutputStream(file.toPath(), StandardOpenOption.WRITE); + OutputStream out = config.isCompressionEnabled() ? new GZIPOutputStream(fout) : fout) { + copyStreams(in, out); + } + + Failsafe.with(retryPolicy).run(() -> uploadData(bucket, key, file)); + } finally { + if (!file.delete()) { + LOG.warn("Failed to delete a temp file: {}", file.getAbsolutePath()); + } } + } + + @Override + public void close() { + client.close(); + } + + public static class Config extends Sender.Config implements Validatable { + private String endpoint; + private String region; + private String awsAccessKeyId; + private String awsSecretAccessKey; + private boolean isCompressionEnabled = true; + + @Min(10) + private int retryIntervalMs = 1000; - public int getRetryInternalMs() - { - return config.getRetryIntervalMs(); + @Min(10) + private int maxRetryIntervalMs = 30000; + + @DecimalMin("1.0") + private float retryFactor = 2; + + @Min(0) + private int retryMax = 10; + + @Min(1024) + private int workBufSize = 8192; + + public String getEndpoint() { + return endpoint; } - public int getMaxRetryInternalMs() - { - return config.getMaxRetryIntervalMs(); + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; } - public float getRetryFactor() - { - return config.getRetryFactor(); + public String getRegion() { + return region; } - public int getRetryMax() - { - return config.getRetryMax(); + public void setRegion(String region) { + this.region = region; } - public int getWorkBufSize() - { - return config.getWorkBufSize(); + public String getAwsAccessKeyId() { + return awsAccessKeyId; } - private void copyStreams(InputStream in, OutputStream out) - throws IOException - { - byte[] buf = new byte[getWorkBufSize()]; - while (true) { - int readLen = in.read(buf); - if (readLen < 0) { - break; - } - out.write(buf, 0, readLen); - } + public void setAwsAccessKeyId(String awsAccessKeyId) { + this.awsAccessKeyId = awsAccessKeyId; } - private void uploadData(String bucket, String key, File file) - { - LOG.debug("Upload data to S3: bucket={}, key={}, fileSize={}", - bucket, key, file.length()); - - try { - PutObjectRequest.Builder builder = PutObjectRequest.builder() - .bucket(bucket) - .key(key); - client.putObject(builder.build(), RequestBody.fromFile(file)); - } - catch (NonRetryableException e) { - throw e; - } - catch (Throwable e) { - throw new RetryableException( - String.format("Failed to upload data. bucket=%s, key=%s", bucket, key), e); - } + public String getAwsSecretAccessKey() { + return awsSecretAccessKey; } - public void send(String bucket, String key, ByteBuffer dataBuffer) - throws IOException - { - File file = File.createTempFile("tmp-fluency-", ".tmp"); - try { - try (InputStream in = new ByteBufferBackedInputStream(dataBuffer); - OutputStream fout = Files.newOutputStream(file.toPath(), StandardOpenOption.WRITE); - OutputStream out = config.isCompressionEnabled() ? new GZIPOutputStream(fout) : fout) { - copyStreams(in, out); - } - - Failsafe.with(retryPolicy).run(() -> uploadData(bucket, key, file)); - } - finally { - if (!file.delete()) { - LOG.warn("Failed to delete a temp file: {}", file.getAbsolutePath()); - } - } + public void setAwsSecretAccessKey(String awsSecretAccessKey) { + this.awsSecretAccessKey = awsSecretAccessKey; } - @Override - public void close() - { - client.close(); + public boolean isCompressionEnabled() { + return isCompressionEnabled; + } + + public void setCompressionEnabled(boolean isCompressionEnabled) { + this.isCompressionEnabled = isCompressionEnabled; + } + + public int getRetryIntervalMs() { + return retryIntervalMs; + } + + public void setRetryIntervalMs(int retryIntervalMs) { + this.retryIntervalMs = retryIntervalMs; + } + + public int getMaxRetryIntervalMs() { + return maxRetryIntervalMs; + } + + public void setMaxRetryIntervalMs(int maxRetryIntervalMs) { + this.maxRetryIntervalMs = maxRetryIntervalMs; } - public static class Config - extends Sender.Config - implements Validatable - { - private String endpoint; - private String region; - private String awsAccessKeyId; - private String awsSecretAccessKey; - private boolean isCompressionEnabled = true; - - @Min(10) - private int retryIntervalMs = 1000; - @Min(10) - private int maxRetryIntervalMs = 30000; - @DecimalMin("1.0") - private float retryFactor = 2; - @Min(0) - private int retryMax = 10; - @Min(1024) - private int workBufSize = 8192; - - public String getEndpoint() - { - return endpoint; - } - - public void setEndpoint(String endpoint) - { - this.endpoint = endpoint; - } - - public String getRegion() - { - return region; - } - - public void setRegion(String region) - { - this.region = region; - } - - public String getAwsAccessKeyId() - { - return awsAccessKeyId; - } - - public void setAwsAccessKeyId(String awsAccessKeyId) - { - this.awsAccessKeyId = awsAccessKeyId; - } - - public String getAwsSecretAccessKey() - { - return awsSecretAccessKey; - } - - public void setAwsSecretAccessKey(String awsSecretAccessKey) - { - this.awsSecretAccessKey = awsSecretAccessKey; - } - - public boolean isCompressionEnabled() - { - return isCompressionEnabled; - } - - public void setCompressionEnabled(boolean isCompressionEnabled) - { - this.isCompressionEnabled = isCompressionEnabled; - } - - public int getRetryIntervalMs() - { - return retryIntervalMs; - } - - public void setRetryIntervalMs(int retryIntervalMs) - { - this.retryIntervalMs = retryIntervalMs; - } - - public int getMaxRetryIntervalMs() - { - return maxRetryIntervalMs; - } - - public void setMaxRetryIntervalMs(int maxRetryIntervalMs) - { - this.maxRetryIntervalMs = maxRetryIntervalMs; - } - - public float getRetryFactor() - { - return retryFactor; - } - - public void setRetryFactor(float retryFactor) - { - this.retryFactor = retryFactor; - } - - public int getRetryMax() - { - return retryMax; - } - - public void setRetryMax(int retryMax) - { - this.retryMax = retryMax; - } - - public int getWorkBufSize() - { - return workBufSize; - } - - public void setWorkBufSize(int workBufSize) - { - this.workBufSize = workBufSize; - } - - @Override - public String toString() - { - return "Config{" + - "endpoint='" + endpoint + '\'' + - ", region='" + region + '\'' + - ", isCompressionEnabled=" + isCompressionEnabled + - ", retryIntervalMs=" + retryIntervalMs + - ", maxRetryIntervalMs=" + maxRetryIntervalMs + - ", retryFactor=" + retryFactor + - ", retryMax=" + retryMax + - ", workBufSize=" + workBufSize + - "} " + super.toString(); - } - - void validateValues() - { - validate(); - } + public float getRetryFactor() { + return retryFactor; + } + + public void setRetryFactor(float retryFactor) { + this.retryFactor = retryFactor; + } + + public int getRetryMax() { + return retryMax; + } + + public void setRetryMax(int retryMax) { + this.retryMax = retryMax; + } + + public int getWorkBufSize() { + return workBufSize; + } + + public void setWorkBufSize(int workBufSize) { + this.workBufSize = workBufSize; + } + + @Override + public String toString() { + return "Config{" + + "endpoint='" + + endpoint + + '\'' + + ", region='" + + region + + '\'' + + ", isCompressionEnabled=" + + isCompressionEnabled + + ", retryIntervalMs=" + + retryIntervalMs + + ", maxRetryIntervalMs=" + + maxRetryIntervalMs + + ", retryFactor=" + + retryFactor + + ", retryMax=" + + retryMax + + ", workBufSize=" + + workBufSize + + "} " + + super.toString(); + } + void validateValues() { + validate(); } + } } diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/AwsS3RecordFormatter.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/AwsS3RecordFormatter.java index a4979677..94633edd 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/AwsS3RecordFormatter.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/AwsS3RecordFormatter.java @@ -18,7 +18,4 @@ import org.komamitsu.fluency.recordformat.RecordFormatter; -public interface AwsS3RecordFormatter - extends RecordFormatter -{ -} +public interface AwsS3RecordFormatter extends RecordFormatter {} diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/CsvRecordFormatter.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/CsvRecordFormatter.java index 15569f98..e111165e 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/CsvRecordFormatter.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/CsvRecordFormatter.java @@ -16,191 +16,169 @@ package org.komamitsu.fluency.aws.s3.recordformat; +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; import org.komamitsu.fluency.recordformat.AbstractRecordFormatter; +import org.komamitsu.fluency.recordformat.RecordFormatter; import org.komamitsu.fluency.recordformat.recordaccessor.MapRecordAccessor; import org.komamitsu.fluency.recordformat.recordaccessor.MessagePackRecordAccessor; import org.komamitsu.fluency.recordformat.recordaccessor.RecordAccessor; -import org.komamitsu.fluency.recordformat.RecordFormatter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayOutputStream; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Map; +public class CsvRecordFormatter extends AbstractRecordFormatter implements AwsS3RecordFormatter { + private static final Logger LOG = LoggerFactory.getLogger(CsvRecordFormatter.class); + private final Config config; + private final byte[] delimiter; + private final byte[] quote; + private final byte[] lineBreak; + + public CsvRecordFormatter() { + this(new Config()); + } + + public CsvRecordFormatter(Config config) { + super(config); + config.validateValues(); + this.config = config; + + String delimiterInConfig = config.getDelimiter(); + if (delimiterInConfig == null) { + delimiter = null; + } else { + delimiter = delimiterInConfig.getBytes(StandardCharsets.UTF_8); + } -public class CsvRecordFormatter - extends AbstractRecordFormatter - implements AwsS3RecordFormatter -{ - private static final Logger LOG = LoggerFactory.getLogger(CsvRecordFormatter.class); - private final Config config; - private final byte[] delimiter; - private final byte[] quote; - private final byte[] lineBreak; - - public CsvRecordFormatter() - { - this(new Config()); + String quoteInConfig = config.getQuote(); + if (quoteInConfig == null) { + quote = null; + } else { + quote = quoteInConfig.getBytes(StandardCharsets.UTF_8); } - public CsvRecordFormatter(Config config) - { - super(config); - config.validateValues(); - this.config = config; + lineBreak = new byte[] {0x0A}; + } + + private byte[] formatInternal(RecordAccessor recordAccessor) { + boolean isFirst = true; + ByteArrayOutputStream output = new ByteArrayOutputStream(); + for (String columnName : config.getColumnNames()) { + if (delimiter != null) { + if (isFirst) { + isFirst = false; + } else { + output.write(delimiter, 0, delimiter.length); + } + } + + String value = recordAccessor.getAsString(columnName); + if (value == null) { + continue; + } + + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + + if (quote == null) { + output.write(bytes, 0, bytes.length); + } else { + output.write(quote, 0, quote.length); + output.write(bytes, 0, bytes.length); + output.write(quote, 0, quote.length); + } + } + output.write(lineBreak, 0, lineBreak.length); + + return output.toByteArray(); + } + + @Override + public byte[] format(String tag, Object timestamp, Map data) { + try { + RecordAccessor recordAccessor = new MapRecordAccessor(data); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return formatInternal(recordAccessor); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a Map record: cause=%s, tag=%s, timestamp=%s, recordCount=%d", + e.getMessage(), tag, timestamp, data.size())); + throw e; + } + } + + @Override + public byte[] formatFromMessagePack( + String tag, Object timestamp, byte[] mapValue, int offset, int len) { + try { + RecordAccessor recordAccessor = + new MessagePackRecordAccessor(ByteBuffer.wrap(mapValue, offset, len)); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return formatInternal(recordAccessor); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, offset=%d, len=%d", + e.getMessage(), tag, timestamp, offset, len)); + throw e; + } + } + + @Override + public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) { + try { + RecordAccessor recordAccessor = new MessagePackRecordAccessor(mapValue); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return formatInternal(recordAccessor); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, bytebuf=%s", + e.getMessage(), tag, timestamp, mapValue)); + throw e; + } + } - String delimiterInConfig = config.getDelimiter(); - if (delimiterInConfig == null) { - delimiter = null; - } - else { - delimiter = delimiterInConfig.getBytes(StandardCharsets.UTF_8); - } + @Override + public String formatName() { + return "csv"; + } - String quoteInConfig = config.getQuote(); - if (quoteInConfig == null) { - quote = null; - } - else { - quote = quoteInConfig.getBytes(StandardCharsets.UTF_8); - } + public static class Config extends RecordFormatter.Config { + private List columnNames; + private String quoteString; + private String delimiter = ","; - lineBreak = new byte[] { 0x0A }; + public List getColumnNames() { + return columnNames; } - private byte[] formatInternal(RecordAccessor recordAccessor) - { - boolean isFirst = true; - ByteArrayOutputStream output = new ByteArrayOutputStream(); - for (String columnName : config.getColumnNames()) { - if (delimiter != null) { - if (isFirst) { - isFirst = false; - } - else { - output.write(delimiter, 0, delimiter.length); - } - } - - String value = recordAccessor.getAsString(columnName); - if (value == null) { - continue; - } - - byte[] bytes = value.getBytes(StandardCharsets.UTF_8); - - if (quote == null) { - output.write(bytes, 0, bytes.length); - } - else { - output.write(quote, 0, quote.length); - output.write(bytes, 0, bytes.length); - output.write(quote, 0, quote.length); - } - } - output.write(lineBreak, 0, lineBreak.length); - - return output.toByteArray(); + public void setColumnNames(List columnNames) { + this.columnNames = columnNames; } - @Override - public byte[] format(String tag, Object timestamp, Map data) - { - try { - RecordAccessor recordAccessor = new MapRecordAccessor(data); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return formatInternal(recordAccessor); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a Map record: cause=%s, tag=%s, timestamp=%s, recordCount=%d", - e.getMessage(), tag, timestamp, data.size())); - throw e; - } + public String getQuote() { + return quoteString; } - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, byte[] mapValue, int offset, int len) - { - try { - RecordAccessor recordAccessor = new MessagePackRecordAccessor(ByteBuffer.wrap(mapValue, offset, len)); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return formatInternal(recordAccessor); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, offset=%d, len=%d", - e.getMessage(), tag, timestamp, offset, len)); - throw e; - } + public void setQuote(String quoteString) { + this.quoteString = quoteString; } - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) - { - try { - RecordAccessor recordAccessor = new MessagePackRecordAccessor(mapValue); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return formatInternal(recordAccessor); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, bytebuf=%s", - e.getMessage(), tag, timestamp, mapValue)); - throw e; - } + public String getDelimiter() { + return delimiter; } - @Override - public String formatName() - { - return "csv"; + public void setDelimiter(String delimiter) { + this.delimiter = delimiter; } - public static class Config - extends RecordFormatter.Config - { - private List columnNames; - private String quoteString; - private String delimiter = ","; - - public List getColumnNames() - { - return columnNames; - } - - public void setColumnNames(List columnNames) - { - this.columnNames = columnNames; - } - - public String getQuote() - { - return quoteString; - } - - public void setQuote(String quoteString) - { - this.quoteString = quoteString; - } - - public String getDelimiter() - { - return delimiter; - } - - public void setDelimiter(String delimiter) - { - this.delimiter = delimiter; - } - - public void validateValues() - { - if (columnNames == null || columnNames.isEmpty()) { - throw new IllegalArgumentException("Column names should be empty"); - } - } + public void validateValues() { + if (columnNames == null || columnNames.isEmpty()) { + throw new IllegalArgumentException("Column names should be empty"); + } } + } } diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/JsonlRecordFormatter.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/JsonlRecordFormatter.java index a384a9d9..1025c664 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/JsonlRecordFormatter.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/JsonlRecordFormatter.java @@ -17,95 +17,83 @@ package org.komamitsu.fluency.aws.s3.recordformat; import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Map; import org.komamitsu.fluency.recordformat.AbstractRecordFormatter; +import org.komamitsu.fluency.recordformat.RecordFormatter; import org.komamitsu.fluency.recordformat.recordaccessor.MapRecordAccessor; import org.komamitsu.fluency.recordformat.recordaccessor.MessagePackRecordAccessor; -import org.komamitsu.fluency.recordformat.RecordFormatter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.Map; - -public class JsonlRecordFormatter - extends AbstractRecordFormatter - implements AwsS3RecordFormatter -{ - private static final Logger LOG = LoggerFactory.getLogger(JsonlRecordFormatter.class); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); +public class JsonlRecordFormatter extends AbstractRecordFormatter implements AwsS3RecordFormatter { + private static final Logger LOG = LoggerFactory.getLogger(JsonlRecordFormatter.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - public JsonlRecordFormatter() - { - this(new Config()); - } + public JsonlRecordFormatter() { + this(new Config()); + } - public JsonlRecordFormatter(Config config) - { - super(config); - } + public JsonlRecordFormatter(Config config) { + super(config); + } - private byte[] appendNewLine(String json) - { - return (json + "\n").getBytes(StandardCharsets.UTF_8); - } + private byte[] appendNewLine(String json) { + return (json + "\n").getBytes(StandardCharsets.UTF_8); + } - @Override - public byte[] format(String tag, Object timestamp, Map data) - { - try { - MapRecordAccessor recordAccessor = new MapRecordAccessor(data); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return appendNewLine(recordAccessor.toJson(OBJECT_MAPPER)); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a Map record: cause=%s, tag=%s, timestamp=%s, recordCount=%d", - e.getMessage(), tag, timestamp, data.size())); - throw e; - } + @Override + public byte[] format(String tag, Object timestamp, Map data) { + try { + MapRecordAccessor recordAccessor = new MapRecordAccessor(data); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return appendNewLine(recordAccessor.toJson(OBJECT_MAPPER)); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a Map record: cause=%s, tag=%s, timestamp=%s, recordCount=%d", + e.getMessage(), tag, timestamp, data.size())); + throw e; } + } - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, byte[] mapValue, int offset, int len) - { - try { - MessagePackRecordAccessor recordAccessor = new MessagePackRecordAccessor(ByteBuffer.wrap(mapValue, offset, len)); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return appendNewLine(recordAccessor.toJson(OBJECT_MAPPER)); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, offset=%d, len=%d", - e.getMessage(), tag, timestamp, offset, len)); - throw e; - } + @Override + public byte[] formatFromMessagePack( + String tag, Object timestamp, byte[] mapValue, int offset, int len) { + try { + MessagePackRecordAccessor recordAccessor = + new MessagePackRecordAccessor(ByteBuffer.wrap(mapValue, offset, len)); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return appendNewLine(recordAccessor.toJson(OBJECT_MAPPER)); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, offset=%d, len=%d", + e.getMessage(), tag, timestamp, offset, len)); + throw e; } + } - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) - { - try { - MessagePackRecordAccessor recordAccessor = new MessagePackRecordAccessor(mapValue); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return appendNewLine(recordAccessor.toJson(OBJECT_MAPPER)); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, bytebuf=%s", - e.getMessage(), tag, timestamp, mapValue)); - throw e; - } + @Override + public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) { + try { + MessagePackRecordAccessor recordAccessor = new MessagePackRecordAccessor(mapValue); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return appendNewLine(recordAccessor.toJson(OBJECT_MAPPER)); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, bytebuf=%s", + e.getMessage(), tag, timestamp, mapValue)); + throw e; } + } - @Override - public String formatName() - { - return "jsonl"; - } + @Override + public String formatName() { + return "jsonl"; + } - public static class Config - extends RecordFormatter.Config - { - } + public static class Config extends RecordFormatter.Config {} } diff --git a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/MessagePackRecordFormatter.java b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/MessagePackRecordFormatter.java index 0e8e06b9..227ad1e2 100644 --- a/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/MessagePackRecordFormatter.java +++ b/fluency-aws-s3/src/main/java/org/komamitsu/fluency/aws/s3/recordformat/MessagePackRecordFormatter.java @@ -16,23 +16,17 @@ package org.komamitsu.fluency.aws.s3.recordformat; - public class MessagePackRecordFormatter - extends org.komamitsu.fluency.recordformat.MessagePackRecordFormatter - implements AwsS3RecordFormatter -{ - public MessagePackRecordFormatter() - { - this(new Config()); - } + extends org.komamitsu.fluency.recordformat.MessagePackRecordFormatter + implements AwsS3RecordFormatter { + public MessagePackRecordFormatter() { + this(new Config()); + } - public MessagePackRecordFormatter(Config config) - { - super(config); - } + public MessagePackRecordFormatter(Config config) { + super(config); + } - public static class Config - extends org.komamitsu.fluency.recordformat.MessagePackRecordFormatter.Config - { - } + public static class Config + extends org.komamitsu.fluency.recordformat.MessagePackRecordFormatter.Config {} } diff --git a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/FluencyBuilderForAwsS3Test.java b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/FluencyBuilderForAwsS3Test.java index 54146bf6..d1dd779b 100644 --- a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/FluencyBuilderForAwsS3Test.java +++ b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/FluencyBuilderForAwsS3Test.java @@ -16,7 +16,23 @@ package org.komamitsu.fluency.aws.s3; +import static java.time.ZoneId.SHORT_IDS; +import static java.time.ZoneOffset.UTC; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + import com.google.common.collect.ImmutableList; +import java.time.ZoneId; +import java.time.ZoneOffset; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -36,198 +52,191 @@ import org.mockito.ArgumentCaptor; import software.amazon.awssdk.regions.Region; -import java.time.ZoneId; -import java.time.ZoneOffset; - -import static java.time.ZoneId.SHORT_IDS; -import static java.time.ZoneOffset.UTC; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; +class FluencyBuilderForAwsS3Test { + private FluencyBuilderForAwsS3 builderWithDefaultConfig; + private FluencyBuilderForAwsS3 builderWithCustomConfig; + private Fluency fluency; + + @BeforeEach + void setUp() { + fluency = mock(Fluency.class); + + builderWithDefaultConfig = spy(new FluencyBuilderForAwsS3()); + builderWithDefaultConfig.setFormatCsvColumnNames(ImmutableList.of("dummy")); + doReturn(fluency) + .when(builderWithDefaultConfig) + .createFluency( + any(RecordFormatter.class), + any(Ingester.class), + any(Buffer.Config.class), + any(Flusher.Config.class)); -class FluencyBuilderForAwsS3Test -{ - private FluencyBuilderForAwsS3 builderWithDefaultConfig; - private FluencyBuilderForAwsS3 builderWithCustomConfig; - private Fluency fluency; - - @BeforeEach - void setUp() { - fluency = mock(Fluency.class); - - builderWithDefaultConfig = spy(new FluencyBuilderForAwsS3()); - builderWithDefaultConfig.setFormatCsvColumnNames(ImmutableList.of("dummy")); - doReturn(fluency).when(builderWithDefaultConfig) - .createFluency( - any(RecordFormatter.class), - any(Ingester.class), - any(Buffer.Config.class), - any(Flusher.Config.class)); - - { - FluencyBuilderForAwsS3 builder = new FluencyBuilderForAwsS3(); - builder.setAwsEndpoint("https://foo.bar.org"); - builder.setAwsRegion("us-east-1"); - builder.setCompressionEnabled(false); - builder.setAwsAccessKeyId("ACCESSKEYID"); - builder.setAwsSecretAccessKey("SECRETACCESSKEY"); - builder.setBufferChunkInitialSize(2 * 1024 * 1024); - builder.setBufferChunkRetentionSize(9 * 1024 * 1024); - builder.setBufferChunkRetentionTimeMillis(1234); - builder.setMaxBufferSize(42 * 1024 * 1024L); - builder.setS3KeyPrefix("mydata"); - builder.setS3KeySuffix(".zzz"); - builder.setS3KeyTimeZoneId(ZoneOffset.of("JST", SHORT_IDS)); - builder.setSenderRetryMax(4); - builder.setSenderMaxRetryIntervalMillis(65432); - builder.setSenderRetryIntervalMillis(543); - builder.setSenderRetryFactor(1.234f); - builder.setSenderWorkBufSize(99 * 1024); - builderWithCustomConfig = spy(builder); - } - doReturn(fluency).when(builderWithCustomConfig) - .createFluency( - any(RecordFormatter.class), - any(Ingester.class), - any(Buffer.Config.class), - any(Flusher.Config.class)); + FluencyBuilderForAwsS3 builder = new FluencyBuilderForAwsS3(); + builder.setAwsEndpoint("https://foo.bar.org"); + builder.setAwsRegion("us-east-1"); + builder.setCompressionEnabled(false); + builder.setAwsAccessKeyId("ACCESSKEYID"); + builder.setAwsSecretAccessKey("SECRETACCESSKEY"); + builder.setBufferChunkInitialSize(2 * 1024 * 1024); + builder.setBufferChunkRetentionSize(9 * 1024 * 1024); + builder.setBufferChunkRetentionTimeMillis(1234); + builder.setMaxBufferSize(42 * 1024 * 1024L); + builder.setS3KeyPrefix("mydata"); + builder.setS3KeySuffix(".zzz"); + builder.setS3KeyTimeZoneId(ZoneOffset.of("JST", SHORT_IDS)); + builder.setSenderRetryMax(4); + builder.setSenderMaxRetryIntervalMillis(65432); + builder.setSenderRetryIntervalMillis(543); + builder.setSenderRetryFactor(1.234f); + builder.setSenderWorkBufSize(99 * 1024); + builderWithCustomConfig = spy(builder); } - - @ParameterizedTest - @EnumSource(FluencyBuilderForAwsS3.FormatType.class) - void buildWithDefaultConfig(FluencyBuilderForAwsS3.FormatType formatType) - { - FluencyBuilderForAwsS3 builder = builderWithDefaultConfig; - AwsS3Sender sender = mock(AwsS3Sender.class); - doReturn(sender).when(builder).createSender(any(AwsS3Sender.Config.class)); - - builder.setFormatType(formatType); - Fluency fluency = builder.build(); - assertEquals(fluency, this.fluency); - - ArgumentCaptor configArgumentCaptor = ArgumentCaptor.forClass(AwsS3Sender.Config.class); - verify(builder, times(1)).createSender(configArgumentCaptor.capture()); - AwsS3Sender.Config senderConfig = configArgumentCaptor.getValue(); - assertNull(senderConfig.getEndpoint()); - assertNull(senderConfig.getRegion()); - assertNull(senderConfig.getAwsAccessKeyId()); - assertNull(senderConfig.getAwsSecretAccessKey()); - assertEquals(10, senderConfig.getRetryMax()); - assertEquals(1000, senderConfig.getRetryIntervalMs()); - assertEquals(30000, senderConfig.getMaxRetryIntervalMs()); - assertEquals(2.0, senderConfig.getRetryFactor()); - assertEquals(8192, senderConfig.getWorkBufSize()); - assertTrue(senderConfig.isCompressionEnabled()); - - ArgumentCaptor recordFormatterArgumentCaptor = ArgumentCaptor.forClass(RecordFormatter.class); - ArgumentCaptor ingesterArgumentCaptor = ArgumentCaptor.forClass(Ingester.class); - verify(builder, times(1)).buildFromIngester( - recordFormatterArgumentCaptor.capture(), ingesterArgumentCaptor.capture()); - - RecordFormatter recordFormatter = recordFormatterArgumentCaptor.getValue(); - Class expectedAwsS3RecordFormatter = null; - String expectedS3KeySuffix = null; - switch (formatType) { - case MESSAGE_PACK: - expectedAwsS3RecordFormatter = MessagePackRecordFormatter.class; - expectedS3KeySuffix = ".msgpack.gz"; - break; - case JSONL: - expectedAwsS3RecordFormatter = JsonlRecordFormatter.class; - expectedS3KeySuffix = ".jsonl.gz"; - break; - case CSV: - expectedAwsS3RecordFormatter = CsvRecordFormatter.class; - expectedS3KeySuffix = ".csv.gz"; - break; - } - assertEquals(expectedAwsS3RecordFormatter, recordFormatter.getClass()); - - AwsS3Ingester ingester = (AwsS3Ingester) ingesterArgumentCaptor.getValue(); - assertEquals(sender, ingester.getSender()); - DefaultS3DestinationDecider destinationDecider = - (DefaultS3DestinationDecider) ingester.getS3DestinationDecider(); - assertNull(destinationDecider.getKeyPrefix()); - assertEquals(expectedS3KeySuffix, destinationDecider.getKeySuffix()); - assertEquals(UTC, destinationDecider.getZoneId()); - - ArgumentCaptor bufferConfigArgumentCaptor = ArgumentCaptor.forClass(Buffer.Config.class); - ArgumentCaptor flusherConfigArgumentCaptor = ArgumentCaptor.forClass(Flusher.Config.class); - verify(builder, times(1)).createFluency( - eq(recordFormatter), - eq(ingester), - bufferConfigArgumentCaptor.capture(), - flusherConfigArgumentCaptor.capture()); - - Buffer.Config bufferConfig = bufferConfigArgumentCaptor.getValue(); - assertEquals(512 * 1024 * 1024, bufferConfig.getMaxBufferSize()); - assertEquals(4 * 1024 * 1024, bufferConfig.getChunkInitialSize()); - assertEquals(64 * 1024 * 1024, bufferConfig.getChunkRetentionSize()); - assertEquals(30 * 1000, bufferConfig.getChunkRetentionTimeMillis()); - } - - @Test - void buildWithCustomConfig() - { - FluencyBuilderForAwsS3 builder = builderWithCustomConfig; - AwsS3Sender sender = mock(AwsS3Sender.class); - doReturn(sender).when(builder).createSender(any(AwsS3Sender.Config.class)); - - builder.setFormatType(FluencyBuilderForAwsS3.FormatType.JSONL); - Fluency fluency = builder.build(); - assertEquals(fluency, this.fluency); - - ArgumentCaptor configArgumentCaptor = ArgumentCaptor.forClass(AwsS3Sender.Config.class); - verify(builder, times(1)).createSender(configArgumentCaptor.capture()); - AwsS3Sender.Config senderConfig = configArgumentCaptor.getValue(); - assertEquals("https://foo.bar.org", senderConfig.getEndpoint()); - assertEquals(Region.US_EAST_1.toString(), senderConfig.getRegion()); - assertEquals("ACCESSKEYID", senderConfig.getAwsAccessKeyId()); - assertEquals("SECRETACCESSKEY", senderConfig.getAwsSecretAccessKey()); - assertEquals(4, senderConfig.getRetryMax()); - assertEquals(543, senderConfig.getRetryIntervalMs()); - assertEquals(65432, senderConfig.getMaxRetryIntervalMs()); - assertEquals(1.234f, senderConfig.getRetryFactor()); - assertEquals(99 * 1024, senderConfig.getWorkBufSize()); - assertFalse(senderConfig.isCompressionEnabled()); - - ArgumentCaptor recordFormatterArgumentCaptor = ArgumentCaptor.forClass(RecordFormatter.class); - ArgumentCaptor ingesterArgumentCaptor = ArgumentCaptor.forClass(Ingester.class); - verify(builder, times(1)).buildFromIngester( - recordFormatterArgumentCaptor.capture(), ingesterArgumentCaptor.capture()); - - RecordFormatter recordFormatter = recordFormatterArgumentCaptor.getValue(); - assertTrue(recordFormatter instanceof JsonlRecordFormatter); - - AwsS3Ingester ingester = (AwsS3Ingester) ingesterArgumentCaptor.getValue(); - assertEquals(sender, ingester.getSender()); - DefaultS3DestinationDecider destinationDecider = - (DefaultS3DestinationDecider) ingester.getS3DestinationDecider(); - assertEquals("mydata", destinationDecider.getKeyPrefix()); - assertEquals(".zzz", destinationDecider.getKeySuffix()); - assertEquals(ZoneId.of("JST", SHORT_IDS), destinationDecider.getZoneId()); - - ArgumentCaptor bufferConfigArgumentCaptor = ArgumentCaptor.forClass(Buffer.Config.class); - ArgumentCaptor flusherConfigArgumentCaptor = ArgumentCaptor.forClass(Flusher.Config.class); - verify(builder, times(1)).createFluency( - eq(recordFormatter), - eq(ingester), - bufferConfigArgumentCaptor.capture(), - flusherConfigArgumentCaptor.capture()); - - Buffer.Config bufferConfig = bufferConfigArgumentCaptor.getValue(); - assertEquals(42 * 1024 * 1024, bufferConfig.getMaxBufferSize()); - assertEquals(2 * 1024 * 1024, bufferConfig.getChunkInitialSize()); - assertEquals(9 * 1024 * 1024, bufferConfig.getChunkRetentionSize()); - assertEquals(1234, bufferConfig.getChunkRetentionTimeMillis()); + doReturn(fluency) + .when(builderWithCustomConfig) + .createFluency( + any(RecordFormatter.class), + any(Ingester.class), + any(Buffer.Config.class), + any(Flusher.Config.class)); + } + + @ParameterizedTest + @EnumSource(FluencyBuilderForAwsS3.FormatType.class) + void buildWithDefaultConfig(FluencyBuilderForAwsS3.FormatType formatType) { + FluencyBuilderForAwsS3 builder = builderWithDefaultConfig; + AwsS3Sender sender = mock(AwsS3Sender.class); + doReturn(sender).when(builder).createSender(any(AwsS3Sender.Config.class)); + + builder.setFormatType(formatType); + Fluency fluency = builder.build(); + assertEquals(fluency, this.fluency); + + ArgumentCaptor configArgumentCaptor = + ArgumentCaptor.forClass(AwsS3Sender.Config.class); + verify(builder, times(1)).createSender(configArgumentCaptor.capture()); + AwsS3Sender.Config senderConfig = configArgumentCaptor.getValue(); + assertNull(senderConfig.getEndpoint()); + assertNull(senderConfig.getRegion()); + assertNull(senderConfig.getAwsAccessKeyId()); + assertNull(senderConfig.getAwsSecretAccessKey()); + assertEquals(10, senderConfig.getRetryMax()); + assertEquals(1000, senderConfig.getRetryIntervalMs()); + assertEquals(30000, senderConfig.getMaxRetryIntervalMs()); + assertEquals(2.0, senderConfig.getRetryFactor()); + assertEquals(8192, senderConfig.getWorkBufSize()); + assertTrue(senderConfig.isCompressionEnabled()); + + ArgumentCaptor recordFormatterArgumentCaptor = + ArgumentCaptor.forClass(RecordFormatter.class); + ArgumentCaptor ingesterArgumentCaptor = ArgumentCaptor.forClass(Ingester.class); + verify(builder, times(1)) + .buildFromIngester( + recordFormatterArgumentCaptor.capture(), ingesterArgumentCaptor.capture()); + + RecordFormatter recordFormatter = recordFormatterArgumentCaptor.getValue(); + Class expectedAwsS3RecordFormatter = null; + String expectedS3KeySuffix = null; + switch (formatType) { + case MESSAGE_PACK: + expectedAwsS3RecordFormatter = MessagePackRecordFormatter.class; + expectedS3KeySuffix = ".msgpack.gz"; + break; + case JSONL: + expectedAwsS3RecordFormatter = JsonlRecordFormatter.class; + expectedS3KeySuffix = ".jsonl.gz"; + break; + case CSV: + expectedAwsS3RecordFormatter = CsvRecordFormatter.class; + expectedS3KeySuffix = ".csv.gz"; + break; } + assertEquals(expectedAwsS3RecordFormatter, recordFormatter.getClass()); + + AwsS3Ingester ingester = (AwsS3Ingester) ingesterArgumentCaptor.getValue(); + assertEquals(sender, ingester.getSender()); + DefaultS3DestinationDecider destinationDecider = + (DefaultS3DestinationDecider) ingester.getS3DestinationDecider(); + assertNull(destinationDecider.getKeyPrefix()); + assertEquals(expectedS3KeySuffix, destinationDecider.getKeySuffix()); + assertEquals(UTC, destinationDecider.getZoneId()); + + ArgumentCaptor bufferConfigArgumentCaptor = + ArgumentCaptor.forClass(Buffer.Config.class); + ArgumentCaptor flusherConfigArgumentCaptor = + ArgumentCaptor.forClass(Flusher.Config.class); + verify(builder, times(1)) + .createFluency( + eq(recordFormatter), + eq(ingester), + bufferConfigArgumentCaptor.capture(), + flusherConfigArgumentCaptor.capture()); + + Buffer.Config bufferConfig = bufferConfigArgumentCaptor.getValue(); + assertEquals(512 * 1024 * 1024, bufferConfig.getMaxBufferSize()); + assertEquals(4 * 1024 * 1024, bufferConfig.getChunkInitialSize()); + assertEquals(64 * 1024 * 1024, bufferConfig.getChunkRetentionSize()); + assertEquals(30 * 1000, bufferConfig.getChunkRetentionTimeMillis()); + } + + @Test + void buildWithCustomConfig() { + FluencyBuilderForAwsS3 builder = builderWithCustomConfig; + AwsS3Sender sender = mock(AwsS3Sender.class); + doReturn(sender).when(builder).createSender(any(AwsS3Sender.Config.class)); + + builder.setFormatType(FluencyBuilderForAwsS3.FormatType.JSONL); + Fluency fluency = builder.build(); + assertEquals(fluency, this.fluency); + + ArgumentCaptor configArgumentCaptor = + ArgumentCaptor.forClass(AwsS3Sender.Config.class); + verify(builder, times(1)).createSender(configArgumentCaptor.capture()); + AwsS3Sender.Config senderConfig = configArgumentCaptor.getValue(); + assertEquals("https://foo.bar.org", senderConfig.getEndpoint()); + assertEquals(Region.US_EAST_1.toString(), senderConfig.getRegion()); + assertEquals("ACCESSKEYID", senderConfig.getAwsAccessKeyId()); + assertEquals("SECRETACCESSKEY", senderConfig.getAwsSecretAccessKey()); + assertEquals(4, senderConfig.getRetryMax()); + assertEquals(543, senderConfig.getRetryIntervalMs()); + assertEquals(65432, senderConfig.getMaxRetryIntervalMs()); + assertEquals(1.234f, senderConfig.getRetryFactor()); + assertEquals(99 * 1024, senderConfig.getWorkBufSize()); + assertFalse(senderConfig.isCompressionEnabled()); + + ArgumentCaptor recordFormatterArgumentCaptor = + ArgumentCaptor.forClass(RecordFormatter.class); + ArgumentCaptor ingesterArgumentCaptor = ArgumentCaptor.forClass(Ingester.class); + verify(builder, times(1)) + .buildFromIngester( + recordFormatterArgumentCaptor.capture(), ingesterArgumentCaptor.capture()); + + RecordFormatter recordFormatter = recordFormatterArgumentCaptor.getValue(); + assertTrue(recordFormatter instanceof JsonlRecordFormatter); + + AwsS3Ingester ingester = (AwsS3Ingester) ingesterArgumentCaptor.getValue(); + assertEquals(sender, ingester.getSender()); + DefaultS3DestinationDecider destinationDecider = + (DefaultS3DestinationDecider) ingester.getS3DestinationDecider(); + assertEquals("mydata", destinationDecider.getKeyPrefix()); + assertEquals(".zzz", destinationDecider.getKeySuffix()); + assertEquals(ZoneId.of("JST", SHORT_IDS), destinationDecider.getZoneId()); + + ArgumentCaptor bufferConfigArgumentCaptor = + ArgumentCaptor.forClass(Buffer.Config.class); + ArgumentCaptor flusherConfigArgumentCaptor = + ArgumentCaptor.forClass(Flusher.Config.class); + verify(builder, times(1)) + .createFluency( + eq(recordFormatter), + eq(ingester), + bufferConfigArgumentCaptor.capture(), + flusherConfigArgumentCaptor.capture()); + + Buffer.Config bufferConfig = bufferConfigArgumentCaptor.getValue(); + assertEquals(42 * 1024 * 1024, bufferConfig.getMaxBufferSize()); + assertEquals(2 * 1024 * 1024, bufferConfig.getChunkInitialSize()); + assertEquals(9 * 1024 * 1024, bufferConfig.getChunkRetentionSize()); + assertEquals(1234, bufferConfig.getChunkRetentionTimeMillis()); + } } diff --git a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/AwsS3IngesterTest.java b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/AwsS3IngesterTest.java index b30131d7..b6882a02 100644 --- a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/AwsS3IngesterTest.java +++ b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/AwsS3IngesterTest.java @@ -16,15 +16,6 @@ package org.komamitsu.fluency.aws.s3.ingester; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.aws.s3.ingester.sender.AwsS3Sender; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.time.Instant; - import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -33,42 +24,46 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -class AwsS3IngesterTest -{ - private AwsS3Sender s3Sender; - private S3DestinationDecider destinationDecider; - private AwsS3Ingester ingester; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.aws.s3.ingester.sender.AwsS3Sender; + +class AwsS3IngesterTest { + private AwsS3Sender s3Sender; + private S3DestinationDecider destinationDecider; + private AwsS3Ingester ingester; - @BeforeEach - void setUp() - { - s3Sender = mock(AwsS3Sender.class); + @BeforeEach + void setUp() { + s3Sender = mock(AwsS3Sender.class); - destinationDecider = mock(S3DestinationDecider.class); - doReturn(new S3DestinationDecider.S3Destination("mybucket", "my/key/base.data")) - .when(destinationDecider).decide(anyString(), any(Instant.class)); + destinationDecider = mock(S3DestinationDecider.class); + doReturn(new S3DestinationDecider.S3Destination("mybucket", "my/key/base.data")) + .when(destinationDecider) + .decide(anyString(), any(Instant.class)); - ingester = new AwsS3Ingester(s3Sender, destinationDecider); - } + ingester = new AwsS3Ingester(s3Sender, destinationDecider); + } - @Test - void ingest() - throws IOException - { - ingester.ingest("foo.bar", - ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))); + @Test + void ingest() throws IOException { + ingester.ingest("foo.bar", ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))); - verify(s3Sender, times(1)) - .send(eq("mybucket"), - eq("my/key/base.data"), - eq(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8)))); - } + verify(s3Sender, times(1)) + .send( + eq("mybucket"), + eq("my/key/base.data"), + eq(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8)))); + } - @Test - void close() - { - ingester.close(); + @Test + void close() { + ingester.close(); - verify(s3Sender, times(1)).close(); - } -} \ No newline at end of file + verify(s3Sender, times(1)).close(); + } +} diff --git a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/DefaultS3DestinationDeciderTest.java b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/DefaultS3DestinationDeciderTest.java index 78e96cbc..02c45bb2 100644 --- a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/DefaultS3DestinationDeciderTest.java +++ b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/DefaultS3DestinationDeciderTest.java @@ -16,58 +16,53 @@ package org.komamitsu.fluency.aws.s3.ingester; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; import java.time.ZoneId; import java.time.ZonedDateTime; +import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.*; - -class DefaultS3DestinationDeciderTest -{ - // +09:00 - private static ZoneId TIMEZONE_JST = ZoneId.of("JST", ZoneId.SHORT_IDS); +class DefaultS3DestinationDeciderTest { + // +09:00 + private static ZoneId TIMEZONE_JST = ZoneId.of("JST", ZoneId.SHORT_IDS); - @Test - void decide() - { - DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config(); - config.setKeyPrefix("archives"); - config.setKeySuffix(".testdata"); - DefaultS3DestinationDecider decider = new DefaultS3DestinationDecider(config); - ZonedDateTime time = ZonedDateTime.of(2019, 12, 31, 23, 59, 59, 999999000, TIMEZONE_JST); - S3DestinationDecider.S3Destination destination = - decider.decide("web.access_log", time.toInstant()); - assertEquals("web.access_log", destination.getBucket()); - // JST is 9 hours ahead of UTC - assertEquals("archives/2019/12/31/14/59-59-999999.testdata", destination.getKey()); - } + @Test + void decide() { + DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config(); + config.setKeyPrefix("archives"); + config.setKeySuffix(".testdata"); + DefaultS3DestinationDecider decider = new DefaultS3DestinationDecider(config); + ZonedDateTime time = ZonedDateTime.of(2019, 12, 31, 23, 59, 59, 999999000, TIMEZONE_JST); + S3DestinationDecider.S3Destination destination = + decider.decide("web.access_log", time.toInstant()); + assertEquals("web.access_log", destination.getBucket()); + // JST is 9 hours ahead of UTC + assertEquals("archives/2019/12/31/14/59-59-999999.testdata", destination.getKey()); + } - @Test - void decideWithSpecificTimeZone() - { - DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config(); - config.setKeyPrefix("archives"); - config.setKeySuffix(".testdata"); - config.setZoneId(TIMEZONE_JST); - DefaultS3DestinationDecider decider = new DefaultS3DestinationDecider(config); - ZonedDateTime time = ZonedDateTime.of(2019, 12, 31, 23, 59, 59, 999999000, TIMEZONE_JST); - S3DestinationDecider.S3Destination destination = - decider.decide("web.access_log", time.toInstant()); - assertEquals("web.access_log", destination.getBucket()); - assertEquals("archives/2019/12/31/23/59-59-999999.testdata", destination.getKey()); - } + @Test + void decideWithSpecificTimeZone() { + DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config(); + config.setKeyPrefix("archives"); + config.setKeySuffix(".testdata"); + config.setZoneId(TIMEZONE_JST); + DefaultS3DestinationDecider decider = new DefaultS3DestinationDecider(config); + ZonedDateTime time = ZonedDateTime.of(2019, 12, 31, 23, 59, 59, 999999000, TIMEZONE_JST); + S3DestinationDecider.S3Destination destination = + decider.decide("web.access_log", time.toInstant()); + assertEquals("web.access_log", destination.getBucket()); + assertEquals("archives/2019/12/31/23/59-59-999999.testdata", destination.getKey()); + } - @Test - void decideWithoutPrefixNorSuffix() - { - DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config(); - DefaultS3DestinationDecider decider = new DefaultS3DestinationDecider(config); - ZonedDateTime time = ZonedDateTime.of(2019, 12, 31, 23, 59, 59, 999999000, TIMEZONE_JST); - S3DestinationDecider.S3Destination destination = - decider.decide("web.access_log", time.toInstant()); - assertEquals("web.access_log", destination.getBucket()); - // JST is 9 hours ahead of UTC - assertEquals("2019/12/31/14/59-59-999999", destination.getKey()); - } + @Test + void decideWithoutPrefixNorSuffix() { + DefaultS3DestinationDecider.Config config = new DefaultS3DestinationDecider.Config(); + DefaultS3DestinationDecider decider = new DefaultS3DestinationDecider(config); + ZonedDateTime time = ZonedDateTime.of(2019, 12, 31, 23, 59, 59, 999999000, TIMEZONE_JST); + S3DestinationDecider.S3Destination destination = + decider.decide("web.access_log", time.toInstant()); + assertEquals("web.access_log", destination.getBucket()); + // JST is 9 hours ahead of UTC + assertEquals("2019/12/31/14/59-59-999999", destination.getKey()); + } } diff --git a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/sender/AwsS3SenderTest.java b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/sender/AwsS3SenderTest.java index 66d2c035..11ef07a3 100644 --- a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/sender/AwsS3SenderTest.java +++ b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/ingester/sender/AwsS3SenderTest.java @@ -16,7 +16,24 @@ package org.komamitsu.fluency.aws.s3.ingester.sender; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + import com.google.common.io.ByteStreams; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.GZIPInputStream; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -29,151 +46,129 @@ import software.amazon.awssdk.services.s3.S3ClientBuilder; import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.zip.GZIPInputStream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -class AwsS3SenderTest -{ - @Test - void buildClientWithDefaults() - { - AwsS3Sender.Config config = new AwsS3Sender.Config(); - - S3Client s3Client = mock(S3Client.class); - S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); - doReturn(s3Client).when(s3ClientBuilder).build(); - - new AwsS3Sender(s3ClientBuilder, config); - - verify(s3ClientBuilder, times(1)).build(); - verify(s3ClientBuilder, times(0)).endpointOverride(any()); - verify(s3ClientBuilder, times(0)).region(any()); - verify(s3ClientBuilder, times(0)).credentialsProvider(any()); - } - - @Test - void buildClientWithCustomizedConfig() - { - AwsS3Sender.Config config = new AwsS3Sender.Config(); - config.setEndpoint("https://another.s3endpoi.nt"); - config.setRegion("ap-northeast-1"); - config.setAwsAccessKeyId("foo"); - config.setAwsSecretAccessKey("bar"); - - S3Client s3Client = mock(S3Client.class); - S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); - doReturn(s3Client).when(s3ClientBuilder).build(); - doAnswer(invocation -> { - AwsCredentialsProvider provider = invocation.getArgument(0); - AwsCredentials awsCredentials = provider.resolveCredentials(); - assertEquals("foo", awsCredentials.accessKeyId()); - assertEquals("bar", awsCredentials.secretAccessKey()); - return null; - }).when(s3ClientBuilder).credentialsProvider(any()); - - new AwsS3Sender(s3ClientBuilder, config); - - verify(s3ClientBuilder, times(1)).build(); - verify(s3ClientBuilder, times(1)).endpointOverride(eq(URI.create("https://another.s3endpoi.nt"))); - verify(s3ClientBuilder, times(1)).region(eq(Region.AP_NORTHEAST_1)); - verify(s3ClientBuilder, times(1)).credentialsProvider(any()); - } - - private void testSend( - AwsS3Sender.Config config, - boolean gzipCompressed, - int failures) - throws IOException - { - S3Client s3Client = mock(S3Client.class); - S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); - doReturn(s3Client).when(s3ClientBuilder).build(); - - AtomicInteger retryCount = new AtomicInteger(); - doAnswer(invocation -> { - PutObjectRequest request = invocation.getArgument(0); - - assertEquals("hello.world", request.bucket()); - assertEquals("2345/01/31/23/59-59-99999.data", request.key()); - - RequestBody body = invocation.getArgument(1); - try (InputStream s3In = body.contentStreamProvider().newStream(); - InputStream in = gzipCompressed ? new GZIPInputStream(s3In) : s3In) { +class AwsS3SenderTest { + @Test + void buildClientWithDefaults() { + AwsS3Sender.Config config = new AwsS3Sender.Config(); + + S3Client s3Client = mock(S3Client.class); + S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); + doReturn(s3Client).when(s3ClientBuilder).build(); + + new AwsS3Sender(s3ClientBuilder, config); + + verify(s3ClientBuilder, times(1)).build(); + verify(s3ClientBuilder, times(0)).endpointOverride(any()); + verify(s3ClientBuilder, times(0)).region(any()); + verify(s3ClientBuilder, times(0)).credentialsProvider(any()); + } + + @Test + void buildClientWithCustomizedConfig() { + AwsS3Sender.Config config = new AwsS3Sender.Config(); + config.setEndpoint("https://another.s3endpoi.nt"); + config.setRegion("ap-northeast-1"); + config.setAwsAccessKeyId("foo"); + config.setAwsSecretAccessKey("bar"); + + S3Client s3Client = mock(S3Client.class); + S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); + doReturn(s3Client).when(s3ClientBuilder).build(); + doAnswer( + invocation -> { + AwsCredentialsProvider provider = invocation.getArgument(0); + AwsCredentials awsCredentials = provider.resolveCredentials(); + assertEquals("foo", awsCredentials.accessKeyId()); + assertEquals("bar", awsCredentials.secretAccessKey()); + return null; + }) + .when(s3ClientBuilder) + .credentialsProvider(any()); + + new AwsS3Sender(s3ClientBuilder, config); + + verify(s3ClientBuilder, times(1)).build(); + verify(s3ClientBuilder, times(1)) + .endpointOverride(eq(URI.create("https://another.s3endpoi.nt"))); + verify(s3ClientBuilder, times(1)).region(eq(Region.AP_NORTHEAST_1)); + verify(s3ClientBuilder, times(1)).credentialsProvider(any()); + } + + private void testSend(AwsS3Sender.Config config, boolean gzipCompressed, int failures) + throws IOException { + S3Client s3Client = mock(S3Client.class); + S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); + doReturn(s3Client).when(s3ClientBuilder).build(); + + AtomicInteger retryCount = new AtomicInteger(); + doAnswer( + invocation -> { + PutObjectRequest request = invocation.getArgument(0); + + assertEquals("hello.world", request.bucket()); + assertEquals("2345/01/31/23/59-59-99999.data", request.key()); + + RequestBody body = invocation.getArgument(1); + try (InputStream s3In = body.contentStreamProvider().newStream(); + InputStream in = gzipCompressed ? new GZIPInputStream(s3In) : s3In) { byte[] content = ByteStreams.toByteArray(in); assertEquals("0123456789", new String(content, StandardCharsets.UTF_8)); - } + } - if (retryCount.getAndIncrement() < failures) { + if (retryCount.getAndIncrement() < failures) { throw new RuntimeException("Something happened"); - } - - return null; - }).when(s3Client).putObject(any(PutObjectRequest.class), any(RequestBody.class)); - - AwsS3Sender sender = new AwsS3Sender(s3ClientBuilder, config); - - sender.send("hello.world", "2345/01/31/23/59-59-99999.data", - ByteBuffer.wrap("0123456789".getBytes(StandardCharsets.UTF_8))); - - verify(s3Client, times(failures + 1)).putObject(any(PutObjectRequest.class), any(RequestBody.class)); - - sender.close(); - } - - @ParameterizedTest - @ValueSource(ints={0, 2}) - void send(int failures) - throws IOException - { - AwsS3Sender.Config config = new AwsS3Sender.Config(); - testSend(config, true, failures); - } - - @ParameterizedTest - @ValueSource(ints={0, 2}) - void sendWithoutCompression(int failures) - throws IOException - { - AwsS3Sender.Config config = new AwsS3Sender.Config(); - config.setCompressionEnabled(false); - - testSend(config, false, failures); - } - - @Test - void sendButRetryOver() - { - AwsS3Sender.Config config = new AwsS3Sender.Config(); - config.setRetryMax(2); - assertThrows(RetryableException.class, () -> testSend(config, true, 3)); - } - - @Test - void close() - { - S3Client s3Client = mock(S3Client.class); - S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); - doReturn(s3Client).when(s3ClientBuilder).build(); - - AwsS3Sender sender = new AwsS3Sender(s3ClientBuilder); - sender.close(); - - verify(s3Client, times(1)).close(); - } - -} \ No newline at end of file + } + + return null; + }) + .when(s3Client) + .putObject(any(PutObjectRequest.class), any(RequestBody.class)); + + AwsS3Sender sender = new AwsS3Sender(s3ClientBuilder, config); + + sender.send( + "hello.world", + "2345/01/31/23/59-59-99999.data", + ByteBuffer.wrap("0123456789".getBytes(StandardCharsets.UTF_8))); + + verify(s3Client, times(failures + 1)) + .putObject(any(PutObjectRequest.class), any(RequestBody.class)); + + sender.close(); + } + + @ParameterizedTest + @ValueSource(ints = {0, 2}) + void send(int failures) throws IOException { + AwsS3Sender.Config config = new AwsS3Sender.Config(); + testSend(config, true, failures); + } + + @ParameterizedTest + @ValueSource(ints = {0, 2}) + void sendWithoutCompression(int failures) throws IOException { + AwsS3Sender.Config config = new AwsS3Sender.Config(); + config.setCompressionEnabled(false); + + testSend(config, false, failures); + } + + @Test + void sendButRetryOver() { + AwsS3Sender.Config config = new AwsS3Sender.Config(); + config.setRetryMax(2); + assertThrows(RetryableException.class, () -> testSend(config, true, 3)); + } + + @Test + void close() { + S3Client s3Client = mock(S3Client.class); + S3ClientBuilder s3ClientBuilder = mock(S3ClientBuilder.class); + doReturn(s3Client).when(s3ClientBuilder).build(); + + AwsS3Sender sender = new AwsS3Sender(s3ClientBuilder); + sender.close(); + + verify(s3Client, times(1)).close(); + } +} diff --git a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/recordformat/CsvRecordFormatterTest.java b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/recordformat/CsvRecordFormatterTest.java index c3125344..89ad5fd7 100644 --- a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/recordformat/CsvRecordFormatterTest.java +++ b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/recordformat/CsvRecordFormatterTest.java @@ -16,181 +16,188 @@ package org.komamitsu.fluency.aws.s3.recordformat; +import static org.junit.jupiter.api.Assertions.assertEquals; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; -import org.msgpack.jackson.dataformat.MessagePackFactory; - import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.function.Consumer; import java.util.function.Function; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.msgpack.jackson.dataformat.MessagePackFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; - -class CsvRecordFormatterTest -{ - private static final String TAG = "foodb.bartbl"; - private static final long FUTURE_EPOCH = 4294967296L; - private static final Map RECORD_0 = ImmutableMap.of("name", "first", "age", 42, "email", "hello@world.com"); - private static final Map RECORD_1 = ImmutableMap.of("name", "second", "age", 55, "time", FUTURE_EPOCH, "comment", "zzzzzz"); - private static final Map RECORD_2 = ImmutableMap.of("job", "knight", "name", "third", "age", 99, "ignored", "foobar"); - private static final Consumer CONFIGURATOR_NORMAL = config -> {}; - private static final Consumer CONFIGURATOR_QUOTE = config -> config.setQuote("\""); - private static final Consumer CONFIGURATOR_TAB_DELIM = config -> config.setDelimiter("\t"); - private static final Consumer CONFIGURATOR_QUOTE_TAB_DELIM = config -> { +class CsvRecordFormatterTest { + private static final String TAG = "foodb.bartbl"; + private static final long FUTURE_EPOCH = 4294967296L; + private static final Map RECORD_0 = + ImmutableMap.of("name", "first", "age", 42, "email", "hello@world.com"); + private static final Map RECORD_1 = + ImmutableMap.of("name", "second", "age", 55, "time", FUTURE_EPOCH, "comment", "zzzzzz"); + private static final Map RECORD_2 = + ImmutableMap.of("job", "knight", "name", "third", "age", 99, "ignored", "foobar"); + private static final Consumer CONFIGURATOR_NORMAL = config -> {}; + private static final Consumer CONFIGURATOR_QUOTE = + config -> config.setQuote("\""); + private static final Consumer CONFIGURATOR_TAB_DELIM = + config -> config.setDelimiter("\t"); + private static final Consumer CONFIGURATOR_QUOTE_TAB_DELIM = + config -> { config.setQuote("\""); config.setDelimiter("\t"); - }; - private static final Function QUOTER_NOTHING = - x -> x == null ? "" : x.toString(); - private static final Function QUOTER_QUOTE = - x -> x == null ? "" : "\"" + x + "\""; - private CsvRecordFormatter.Config config; - - enum Option { - NORMAL("%s,%s,%s,%s,,%s,%s\n", - CONFIGURATOR_NORMAL, - QUOTER_NOTHING), - QUOTE("%s,%s,%s,%s,,%s,%s\n", - CONFIGURATOR_QUOTE, - QUOTER_QUOTE), - TAB_DELIM("%s\t%s\t%s\t%s\t\t%s\t%s\n", - CONFIGURATOR_TAB_DELIM, - QUOTER_NOTHING), - QUOTE_AND_TAB_DELIM("%s\t%s\t%s\t%s\t\t%s\t%s\n", - CONFIGURATOR_QUOTE_TAB_DELIM, - QUOTER_QUOTE); - - private final String format; - private final Consumer configurator; - private final Function quoter; - - Option(String format, Consumer configurator, Function quoter) - { - this.format = format; - this.configurator = configurator; - this.quoter = quoter; - } + }; + private static final Function QUOTER_NOTHING = x -> x == null ? "" : x.toString(); + private static final Function QUOTER_QUOTE = + x -> x == null ? "" : "\"" + x + "\""; + private CsvRecordFormatter.Config config; + + enum Option { + NORMAL("%s,%s,%s,%s,,%s,%s\n", CONFIGURATOR_NORMAL, QUOTER_NOTHING), + QUOTE("%s,%s,%s,%s,,%s,%s\n", CONFIGURATOR_QUOTE, QUOTER_QUOTE), + TAB_DELIM("%s\t%s\t%s\t%s\t\t%s\t%s\n", CONFIGURATOR_TAB_DELIM, QUOTER_NOTHING), + QUOTE_AND_TAB_DELIM("%s\t%s\t%s\t%s\t\t%s\t%s\n", CONFIGURATOR_QUOTE_TAB_DELIM, QUOTER_QUOTE); + + private final String format; + private final Consumer configurator; + private final Function quoter; + + Option( + String format, + Consumer configurator, + Function quoter) { + this.format = format; + this.configurator = configurator; + this.quoter = quoter; } - - @BeforeEach - void setUp() - { - config = new CsvRecordFormatter.Config(); - config.setColumnNames(ImmutableList.of("time", "name", "age", "email", "none", "comment", "job")); - } - - private void assertRecord0(byte[] formatted, long expectedTime, Option option) - { - assertEquals( - String.format(option.format, - option.quoter.apply(expectedTime), // time - option.quoter.apply("first"), // name - option.quoter.apply(42), // age - option.quoter.apply("hello@world.com"), // email - option.quoter.apply(null), // comment - option.quoter.apply(null) // job - ), - new String(formatted, StandardCharsets.UTF_8)); - } - - private void assertRecord1(byte[] formatted, long expectedTime, Option option) - { - assertEquals( - String.format(option.format, - option.quoter.apply(expectedTime), // time - option.quoter.apply("second"), // name - option.quoter.apply(55), // age - option.quoter.apply(null), // email - option.quoter.apply("zzzzzz"), // comment - option.quoter.apply(null) // job - ), - new String(formatted, StandardCharsets.UTF_8)); - } - - private void assertRecord2(byte[] formatted, long expectedTime, Option option) - { - assertEquals( - String.format(option.format, - option.quoter.apply(expectedTime), // time - option.quoter.apply("third"), // name - option.quoter.apply(99), // age - option.quoter.apply(null), // email - option.quoter.apply(null), // comment - option.quoter.apply("knight") // job - ), - new String(formatted, StandardCharsets.UTF_8)); - } - - @ParameterizedTest - @EnumSource(Option.class) - void format(Option option) + } + + @BeforeEach + void setUp() { + config = new CsvRecordFormatter.Config(); + config.setColumnNames( + ImmutableList.of("time", "name", "age", "email", "none", "comment", "job")); + } + + private void assertRecord0(byte[] formatted, long expectedTime, Option option) { + assertEquals( + String.format( + option.format, + option.quoter.apply(expectedTime), // time + option.quoter.apply("first"), // name + option.quoter.apply(42), // age + option.quoter.apply("hello@world.com"), // email + option.quoter.apply(null), // comment + option.quoter.apply(null) // job + ), + new String(formatted, StandardCharsets.UTF_8)); + } + + private void assertRecord1(byte[] formatted, long expectedTime, Option option) { + assertEquals( + String.format( + option.format, + option.quoter.apply(expectedTime), // time + option.quoter.apply("second"), // name + option.quoter.apply(55), // age + option.quoter.apply(null), // email + option.quoter.apply("zzzzzz"), // comment + option.quoter.apply(null) // job + ), + new String(formatted, StandardCharsets.UTF_8)); + } + + private void assertRecord2(byte[] formatted, long expectedTime, Option option) { + assertEquals( + String.format( + option.format, + option.quoter.apply(expectedTime), // time + option.quoter.apply("third"), // name + option.quoter.apply(99), // age + option.quoter.apply(null), // email + option.quoter.apply(null), // comment + option.quoter.apply("knight") // job + ), + new String(formatted, StandardCharsets.UTF_8)); + } + + @ParameterizedTest + @EnumSource(Option.class) + void format(Option option) { + option.configurator.accept(config); + CsvRecordFormatter recordFormatter = new CsvRecordFormatter(config); + long now = System.currentTimeMillis() / 1000; + assertRecord0(recordFormatter.format(TAG, now, RECORD_0), now, option); + assertRecord1(recordFormatter.format(TAG, now, RECORD_1), FUTURE_EPOCH, option); + assertRecord2(recordFormatter.format(TAG, now, RECORD_2), now, option); + } + + @ParameterizedTest + @EnumSource(Option.class) + void formatFromMessagePackBytes(Option option) throws IOException { + option.configurator.accept(config); + CsvRecordFormatter recordFormatter = new CsvRecordFormatter(config); + long now = System.currentTimeMillis() / 1000; + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); { - option.configurator.accept(config); - CsvRecordFormatter recordFormatter = new CsvRecordFormatter(config); - long now = System.currentTimeMillis() / 1000; - assertRecord0(recordFormatter.format(TAG, now, RECORD_0), now, option); - assertRecord1(recordFormatter.format(TAG, now, RECORD_1), FUTURE_EPOCH, option); - assertRecord2(recordFormatter.format(TAG, now, RECORD_2), now, option); + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_0); + assertRecord0( + recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now, option); } - - @ParameterizedTest - @EnumSource(Option.class) - void formatFromMessagePackBytes(Option option) - throws IOException { - option.configurator.accept(config); - CsvRecordFormatter recordFormatter = new CsvRecordFormatter(config); - long now = System.currentTimeMillis() / 1000; - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_0); - assertRecord0(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now, option); - } - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_1); - assertRecord1(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), FUTURE_EPOCH, option); - } - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_2); - assertRecord2(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now, option); - } + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_1); + assertRecord1( + recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), + FUTURE_EPOCH, + option); } - - private ByteBuffer convertMapToMessagePackByteBuffer(Map record, boolean isDirect) - throws JsonProcessingException { - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - byte[] bytes = objectMapper.writeValueAsBytes(record); - ByteBuffer byteBuffer; - if (isDirect) { - byteBuffer = ByteBuffer.allocateDirect(bytes.length); - } - else { - byteBuffer = ByteBuffer.allocate(bytes.length); - } - byteBuffer.put(bytes); - byteBuffer.flip(); - return byteBuffer; + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_2); + assertRecord2( + recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now, option); } - - @ParameterizedTest - @EnumSource(Option.class) - void formatFromMessagePackByteBuffer(Option option) - throws IOException - { - option.configurator.accept(config); - CsvRecordFormatter recordFormatter = new CsvRecordFormatter(config); - long now = System.currentTimeMillis() / 1000; - assertRecord0(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_0, false)), now, option); - assertRecord1(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_1, true)), FUTURE_EPOCH, option); - assertRecord2(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_2, true)), now, option); + } + + private ByteBuffer convertMapToMessagePackByteBuffer(Map record, boolean isDirect) + throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + byte[] bytes = objectMapper.writeValueAsBytes(record); + ByteBuffer byteBuffer; + if (isDirect) { + byteBuffer = ByteBuffer.allocateDirect(bytes.length); + } else { + byteBuffer = ByteBuffer.allocate(bytes.length); } -} \ No newline at end of file + byteBuffer.put(bytes); + byteBuffer.flip(); + return byteBuffer; + } + + @ParameterizedTest + @EnumSource(Option.class) + void formatFromMessagePackByteBuffer(Option option) throws IOException { + option.configurator.accept(config); + CsvRecordFormatter recordFormatter = new CsvRecordFormatter(config); + long now = System.currentTimeMillis() / 1000; + assertRecord0( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_0, false)), + now, + option); + assertRecord1( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_1, true)), + FUTURE_EPOCH, + option); + assertRecord2( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_2, true)), + now, + option); + } +} diff --git a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/recordformat/JsonlRecordFormatterTest.java b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/recordformat/JsonlRecordFormatterTest.java index 651a2132..d00a1ce5 100644 --- a/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/recordformat/JsonlRecordFormatterTest.java +++ b/fluency-aws-s3/src/test/java/org/komamitsu/fluency/aws/s3/recordformat/JsonlRecordFormatterTest.java @@ -16,127 +16,123 @@ package org.komamitsu.fluency.aws.s3.recordformat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.msgpack.jackson.dataformat.MessagePackFactory; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Map; +class JsonlRecordFormatterTest { + private static final String TAG = "foodb.bartbl"; + private static final long FUTURE_EPOCH = 4294967296L; + private static final Map RECORD_0 = + ImmutableMap.of("name", "first", "age", 42, "email", "hello@world.com"); + private static final Map RECORD_1 = + ImmutableMap.of("name", "second", "age", 55, "time", FUTURE_EPOCH, "comment", "zzzzzz"); + private static final Map RECORD_2 = + ImmutableMap.of("job", "knight", "name", "third", "age", 99); + private final ObjectMapper objectMapper = new ObjectMapper(); + private JsonlRecordFormatter recordFormatter; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; + @BeforeEach + void setUp() { + recordFormatter = new JsonlRecordFormatter(new JsonlRecordFormatter.Config()); + } -class JsonlRecordFormatterTest -{ - private static final String TAG = "foodb.bartbl"; - private static final long FUTURE_EPOCH = 4294967296L; - private static final Map RECORD_0 = ImmutableMap.of("name", "first", "age", 42, "email", "hello@world.com"); - private static final Map RECORD_1 = ImmutableMap.of("name", "second", "age", 55, "time", FUTURE_EPOCH, "comment", "zzzzzz"); - private static final Map RECORD_2 = ImmutableMap.of("job", "knight", "name", "third", "age", 99); - private final ObjectMapper objectMapper = new ObjectMapper(); - private JsonlRecordFormatter recordFormatter; + private void assertRecord0(byte[] formatted, long expectedTime) throws IOException { + JsonNode jsonNode = objectMapper.readTree(formatted); + assertTrue(jsonNode.isObject()); + assertEquals(4, jsonNode.size()); + assertEquals(expectedTime, jsonNode.get("time").asLong()); + assertEquals("first", jsonNode.get("name").asText()); + assertEquals(42, jsonNode.get("age").asInt()); + assertEquals("hello@world.com", jsonNode.get("email").asText()); + } - @BeforeEach - void setUp() - { - recordFormatter = new JsonlRecordFormatter(new JsonlRecordFormatter.Config()); - } + private void assertRecord1(byte[] formatted, long expectedTime) throws IOException { + JsonNode jsonNode = objectMapper.readTree(formatted); + assertTrue(jsonNode.isObject()); + assertEquals(4, jsonNode.size()); + assertEquals(expectedTime, jsonNode.get("time").asLong()); + assertEquals("second", jsonNode.get("name").asText()); + assertEquals(55, jsonNode.get("age").asInt()); + assertEquals("zzzzzz", jsonNode.get("comment").asText()); + } - private void assertRecord0(byte[] formatted, long expectedTime) - throws IOException - { - JsonNode jsonNode = objectMapper.readTree(formatted); - assertTrue(jsonNode.isObject()); - assertEquals(4, jsonNode.size()); - assertEquals(expectedTime, jsonNode.get("time").asLong()); - assertEquals("first", jsonNode.get("name").asText()); - assertEquals(42, jsonNode.get("age").asInt()); - assertEquals("hello@world.com", jsonNode.get("email").asText()); - } + private void assertRecord2(byte[] formatted, long expectedTime) throws IOException { + JsonNode jsonNode = objectMapper.readTree(formatted); + assertTrue(jsonNode.isObject()); + assertEquals(4, jsonNode.size()); + assertEquals(expectedTime, jsonNode.get("time").asLong()); + assertEquals("third", jsonNode.get("name").asText()); + assertEquals(99, jsonNode.get("age").asInt()); + assertEquals("knight", jsonNode.get("job").asText()); + } - private void assertRecord1(byte[] formatted, long expectedTime) - throws IOException - { - JsonNode jsonNode = objectMapper.readTree(formatted); - assertTrue(jsonNode.isObject()); - assertEquals(4, jsonNode.size()); - assertEquals(expectedTime, jsonNode.get("time").asLong()); - assertEquals("second", jsonNode.get("name").asText()); - assertEquals(55, jsonNode.get("age").asInt()); - assertEquals("zzzzzz", jsonNode.get("comment").asText()); - } + @Test + void format() throws IOException { + long now = System.currentTimeMillis() / 1000; + assertRecord0(recordFormatter.format(TAG, now, RECORD_0), now); + assertRecord1(recordFormatter.format(TAG, now, RECORD_1), FUTURE_EPOCH); + assertRecord2(recordFormatter.format(TAG, now, RECORD_2), now); + } - private void assertRecord2(byte[] formatted, long expectedTime) - throws IOException + @Test + void formatFromMessagePackBytes() throws IOException { + long now = System.currentTimeMillis() / 1000; + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); { - JsonNode jsonNode = objectMapper.readTree(formatted); - assertTrue(jsonNode.isObject()); - assertEquals(4, jsonNode.size()); - assertEquals(expectedTime, jsonNode.get("time").asLong()); - assertEquals("third", jsonNode.get("name").asText()); - assertEquals(99, jsonNode.get("age").asInt()); - assertEquals("knight", jsonNode.get("job").asText()); + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_0); + assertRecord0(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now); } - - @Test - void format() - throws IOException { - long now = System.currentTimeMillis() / 1000; - assertRecord0(recordFormatter.format(TAG, now, RECORD_0), now); - assertRecord1(recordFormatter.format(TAG, now, RECORD_1), FUTURE_EPOCH); - assertRecord2(recordFormatter.format(TAG, now, RECORD_2), now); + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_1); + assertRecord1( + recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), FUTURE_EPOCH); } - - @Test - void formatFromMessagePackBytes() - throws IOException { - long now = System.currentTimeMillis() / 1000; - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_0); - assertRecord0(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now); - } - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_1); - assertRecord1(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), FUTURE_EPOCH); - } - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_2); - assertRecord2(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now); - } + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_2); + assertRecord2(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now); } + } - private ByteBuffer convertMapToMessagePackByteBuffer(Map record, boolean isDirect) - throws JsonProcessingException - { - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - byte[] bytes = objectMapper.writeValueAsBytes(record); - ByteBuffer byteBuffer; - if (isDirect) { - byteBuffer = ByteBuffer.allocateDirect(bytes.length); - } - else { - byteBuffer = ByteBuffer.allocate(bytes.length); - } - byteBuffer.put(bytes); - byteBuffer.flip(); - return byteBuffer; + private ByteBuffer convertMapToMessagePackByteBuffer(Map record, boolean isDirect) + throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + byte[] bytes = objectMapper.writeValueAsBytes(record); + ByteBuffer byteBuffer; + if (isDirect) { + byteBuffer = ByteBuffer.allocateDirect(bytes.length); + } else { + byteBuffer = ByteBuffer.allocate(bytes.length); } + byteBuffer.put(bytes); + byteBuffer.flip(); + return byteBuffer; + } - @Test - void formatFromMessagePackByteBuffer() - throws IOException - { - long now = System.currentTimeMillis() / 1000; - assertRecord0(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_0, false)), now); - assertRecord1(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_1, true)), FUTURE_EPOCH); - assertRecord2(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_2, true)), now); - } -} \ No newline at end of file + @Test + void formatFromMessagePackByteBuffer() throws IOException { + long now = System.currentTimeMillis() / 1000; + assertRecord0( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_0, false)), + now); + assertRecord1( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_1, true)), + FUTURE_EPOCH); + assertRecord2( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_2, true)), + now); + } +} diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/BufferFullException.java b/fluency-core/src/main/java/org/komamitsu/fluency/BufferFullException.java index c0363c4d..213eb775 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/BufferFullException.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/BufferFullException.java @@ -18,11 +18,8 @@ import java.io.IOException; -public class BufferFullException - extends IOException -{ - public BufferFullException(String s) - { - super(s); - } +public class BufferFullException extends IOException { + public BufferFullException(String s) { + super(s); + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/EventTime.java b/fluency-core/src/main/java/org/komamitsu/fluency/EventTime.java index a78e2434..43195026 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/EventTime.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/EventTime.java @@ -20,160 +20,142 @@ import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.fasterxml.jackson.databind.ser.std.StdSerializer; -import org.msgpack.jackson.dataformat.MessagePackExtensionType; -import org.msgpack.jackson.dataformat.MessagePackGenerator; - import java.io.IOException; import java.nio.ByteBuffer; +import org.msgpack.jackson.dataformat.MessagePackExtensionType; +import org.msgpack.jackson.dataformat.MessagePackGenerator; @JsonSerialize(using = EventTime.Serializer.class) -public class EventTime -{ - private final long seconds; - private final long nanoseconds; - - /** - * Constructs an EventTime. - * - * @param seconds the epoch seconds. This should be a 32-bit value. - * @param nanoseconds the nanoseconds. This should be a 32-bit value. - */ - public EventTime(long seconds, long nanoseconds) - { - if (seconds >> 32 != 0) { - throw new IllegalArgumentException("`seconds` should be a 32-bit value"); - } - - if (nanoseconds >> 32 != 0) { - throw new IllegalArgumentException("`nanoseconds` should be a 32-bit value"); - } - - this.seconds = seconds; - this.nanoseconds = nanoseconds; - } - - /** - * Constructs an EventTime. - * - * @param epochSeconds the epoch seconds. This should be a 32-bit value. - */ - public static EventTime fromEpoch(long epochSeconds) - { - return new EventTime(epochSeconds, 0); +public class EventTime { + private final long seconds; + private final long nanoseconds; + + /** + * Constructs an EventTime. + * + * @param seconds the epoch seconds. This should be a 32-bit value. + * @param nanoseconds the nanoseconds. This should be a 32-bit value. + */ + public EventTime(long seconds, long nanoseconds) { + if (seconds >> 32 != 0) { + throw new IllegalArgumentException("`seconds` should be a 32-bit value"); } - /** - * Constructs an EventTime. - * - * @param epochSeconds the epoch seconds. This should be a 32-bit value. - * @param nanoseconds the nanoseconds. This should be a 32-bit value. - */ - public static EventTime fromEpoch(long epochSeconds, long nanoseconds) - { - return new EventTime(epochSeconds, nanoseconds); + if (nanoseconds >> 32 != 0) { + throw new IllegalArgumentException("`nanoseconds` should be a 32-bit value"); } - /** - * Constructs an EventTime. - * - * @param epochMillisecond the epoch milli seconds. - * This should be a 32-bit value. - */ - public static EventTime fromEpochMilli(long epochMillisecond) - { - return new EventTime(epochMillisecond / 1000, (epochMillisecond % 1000) * 1000000); + this.seconds = seconds; + this.nanoseconds = nanoseconds; + } + + /** + * Constructs an EventTime. + * + * @param epochSeconds the epoch seconds. This should be a 32-bit value. + */ + public static EventTime fromEpoch(long epochSeconds) { + return new EventTime(epochSeconds, 0); + } + + /** + * Constructs an EventTime. + * + * @param epochSeconds the epoch seconds. This should be a 32-bit value. + * @param nanoseconds the nanoseconds. This should be a 32-bit value. + */ + public static EventTime fromEpoch(long epochSeconds, long nanoseconds) { + return new EventTime(epochSeconds, nanoseconds); + } + + /** + * Constructs an EventTime. + * + * @param epochMillisecond the epoch milli seconds. This should be a 32-bit value. + */ + public static EventTime fromEpochMilli(long epochMillisecond) { + return new EventTime(epochMillisecond / 1000, (epochMillisecond % 1000) * 1000000); + } + + public long getSeconds() { + return seconds; + } + + public long getNanoseconds() { + return nanoseconds; + } + + /** + * @deprecated As of release 1.9, replaced by {@link #getNanoseconds()} + */ + @Deprecated + public long getNanoSeconds() { + return nanoseconds; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; } - - public long getSeconds() - { - return seconds; + if (!(o instanceof EventTime)) { + return false; } - public long getNanoseconds() - { - return nanoseconds; - } + EventTime eventTime = (EventTime) o; - /** - * @deprecated As of release 1.9, replaced by {@link #getNanoseconds()} - */ - @Deprecated - public long getNanoSeconds() - { - return nanoseconds; + if (seconds != eventTime.seconds) { + return false; } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (!(o instanceof EventTime)) { - return false; - } - - EventTime eventTime = (EventTime) o; - - if (seconds != eventTime.seconds) { - return false; - } - return nanoseconds == eventTime.nanoseconds; + return nanoseconds == eventTime.nanoseconds; + } + + @Override + public int hashCode() { + int result = (int) (seconds ^ (seconds >>> 32)); + result = 31 * result + (int) (nanoseconds ^ (nanoseconds >>> 32)); + return result; + } + + @Override + public String toString() { + return "EventTime{" + "seconds=" + seconds + ", nanoseconds=" + nanoseconds + '}'; + } + + public static class Serializer extends StdSerializer { + public Serializer() { + super(EventTime.class); } - @Override - public int hashCode() - { - int result = (int) (seconds ^ (seconds >>> 32)); - result = 31 * result + (int) (nanoseconds ^ (nanoseconds >>> 32)); - return result; + protected Serializer(Class t) { + super(t); } @Override - public String toString() - { - return "EventTime{" + - "seconds=" + seconds + - ", nanoseconds=" + nanoseconds + - '}'; - } - - public static class Serializer - extends StdSerializer - { - public Serializer() - { - super(EventTime.class); - } - - protected Serializer(Class t) - { - super(t); - } - - @Override - public void serialize(EventTime value, JsonGenerator gen, SerializerProvider provider) - throws IOException - { - if (!(gen instanceof MessagePackGenerator)) { - throw new IllegalStateException("" + - "This class should be serialized by MessagePackGenerator, but `gen` is " + gen.getClass()); - } - - MessagePackGenerator messagePackGenerator = (MessagePackGenerator) gen; - - // https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#eventtime-ext-format - // - // +-------+----+----+----+----+----+----+----+----+----+ - // | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | - // +-------+----+----+----+----+----+----+----+----+----+ - // | D7 | 00 | seconds from epoch| nanosecond | - // +-------+----+----+----+----+----+----+----+----+----+ - // |fixext8|type| 32bits integer BE | 32bits integer BE | - // +-------+----+----+----+----+----+----+----+----+----+ - ByteBuffer buffer = ByteBuffer.allocate(8); - buffer.putInt((int) value.seconds).putInt((int) value.nanoseconds); - messagePackGenerator.writeExtensionType(new MessagePackExtensionType((byte) 0x0, buffer.array())); - } + public void serialize(EventTime value, JsonGenerator gen, SerializerProvider provider) + throws IOException { + if (!(gen instanceof MessagePackGenerator)) { + throw new IllegalStateException( + "" + + "This class should be serialized by MessagePackGenerator, but `gen` is " + + gen.getClass()); + } + + MessagePackGenerator messagePackGenerator = (MessagePackGenerator) gen; + + // https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#eventtime-ext-format + // + // +-------+----+----+----+----+----+----+----+----+----+ + // | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | + // +-------+----+----+----+----+----+----+----+----+----+ + // | D7 | 00 | seconds from epoch| nanosecond | + // +-------+----+----+----+----+----+----+----+----+----+ + // |fixext8|type| 32bits integer BE | 32bits integer BE | + // +-------+----+----+----+----+----+----+----+----+----+ + ByteBuffer buffer = ByteBuffer.allocate(8); + buffer.putInt((int) value.seconds).putInt((int) value.nanoseconds); + messagePackGenerator.writeExtensionType( + new MessagePackExtensionType((byte) 0x0, buffer.array())); } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/Fluency.java b/fluency-core/src/main/java/org/komamitsu/fluency/Fluency.java index 9b5bdf6c..6bc051a3 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/Fluency.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/Fluency.java @@ -16,191 +16,161 @@ package org.komamitsu.fluency; -import org.komamitsu.fluency.buffer.Buffer; -import org.komamitsu.fluency.flusher.Flusher; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.Flushable; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Map; import java.util.concurrent.TimeUnit; +import org.komamitsu.fluency.buffer.Buffer; +import org.komamitsu.fluency.flusher.Flusher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class Fluency - implements Flushable, Closeable -{ - private static final Logger LOG = LoggerFactory.getLogger(Fluency.class); - private final Buffer buffer; - private final Flusher flusher; - private final Emitter emitter = new Emitter(); - - // Fluency has this public constructor, but using FluentBuilder is usually more recommended. - public Fluency(Buffer buffer, Flusher flusher) - { - this.buffer = buffer; - this.flusher = flusher; - } - - public void emit(final String tag, final long timestamp, final Map data) - throws IOException - { - emitter.emit(() -> buffer.append(tag, timestamp, data)); - } - - public void emit(String tag, Map data) - throws IOException - { - emit(tag, System.currentTimeMillis() / 1000, data); - } - - public void emit(final String tag, final EventTime eventTime, final Map data) - throws IOException - { - emitter.emit(() -> buffer.append(tag, eventTime, data)); - } - - public void emit(final String tag, final long timestamp, final byte[] mapValue, final int offset, final int len) - throws IOException - { - emitter.emit(() -> buffer.appendMessagePackMapValue(tag, timestamp, mapValue, offset, len)); - } - - public void emit(String tag, byte[] mapValue, int offset, int len) - throws IOException - { - emit(tag, System.currentTimeMillis() / 1000, mapValue, offset, len); - } - - public void emit(final String tag, final EventTime eventTime, final byte[] mapValue, final int offset, final int len) - throws IOException - { - emitter.emit(() -> buffer.appendMessagePackMapValue(tag, eventTime, mapValue, offset, len)); - } - - public void emit(final String tag, final long timestamp, final ByteBuffer mapValue) - throws IOException - { - emitter.emit(() -> buffer.appendMessagePackMapValue(tag, timestamp, mapValue)); - } - - public void emit(String tag, ByteBuffer mapValue) - throws IOException - { - emit(tag, System.currentTimeMillis() / 1000, mapValue); - } - - public void emit(final String tag, final EventTime eventTime, final ByteBuffer mapValue) - throws IOException - { - emitter.emit(() -> buffer.appendMessagePackMapValue(tag, eventTime, mapValue)); - } - - @Override - public void flush() - throws IOException - { +public class Fluency implements Flushable, Closeable { + private static final Logger LOG = LoggerFactory.getLogger(Fluency.class); + private final Buffer buffer; + private final Flusher flusher; + private final Emitter emitter = new Emitter(); + + // Fluency has this public constructor, but using FluentBuilder is usually more recommended. + public Fluency(Buffer buffer, Flusher flusher) { + this.buffer = buffer; + this.flusher = flusher; + } + + public void emit(final String tag, final long timestamp, final Map data) + throws IOException { + emitter.emit(() -> buffer.append(tag, timestamp, data)); + } + + public void emit(String tag, Map data) throws IOException { + emit(tag, System.currentTimeMillis() / 1000, data); + } + + public void emit(final String tag, final EventTime eventTime, final Map data) + throws IOException { + emitter.emit(() -> buffer.append(tag, eventTime, data)); + } + + public void emit( + final String tag, + final long timestamp, + final byte[] mapValue, + final int offset, + final int len) + throws IOException { + emitter.emit(() -> buffer.appendMessagePackMapValue(tag, timestamp, mapValue, offset, len)); + } + + public void emit(String tag, byte[] mapValue, int offset, int len) throws IOException { + emit(tag, System.currentTimeMillis() / 1000, mapValue, offset, len); + } + + public void emit( + final String tag, + final EventTime eventTime, + final byte[] mapValue, + final int offset, + final int len) + throws IOException { + emitter.emit(() -> buffer.appendMessagePackMapValue(tag, eventTime, mapValue, offset, len)); + } + + public void emit(final String tag, final long timestamp, final ByteBuffer mapValue) + throws IOException { + emitter.emit(() -> buffer.appendMessagePackMapValue(tag, timestamp, mapValue)); + } + + public void emit(String tag, ByteBuffer mapValue) throws IOException { + emit(tag, System.currentTimeMillis() / 1000, mapValue); + } + + public void emit(final String tag, final EventTime eventTime, final ByteBuffer mapValue) + throws IOException { + emitter.emit(() -> buffer.appendMessagePackMapValue(tag, eventTime, mapValue)); + } + + @Override + public void flush() throws IOException { + flusher.flush(); + } + + @Override + public void close() throws IOException { + flusher.close(); + } + + public void clearBackupFiles() { + buffer.clearBackupFiles(); + } + + public long getAllocatedBufferSize() { + return buffer.getAllocatedSize(); + } + + public long getBufferedDataSize() { + return buffer.getBufferedDataSize(); + } + + public boolean isTerminated() { + return flusher.isTerminated(); + } + + public boolean waitUntilAllBufferFlushed(int maxWaitSeconds) throws InterruptedException { + int intervalMilli = 500; + for (int i = 0; i < maxWaitSeconds * (1000 / intervalMilli); i++) { + long bufferedDataSize = getBufferedDataSize(); + LOG.debug("Waiting for flushing all buffer: {}", bufferedDataSize); + if (bufferedDataSize == 0) { + return true; + } + TimeUnit.MILLISECONDS.sleep(intervalMilli); + } + LOG.warn("Buffered data still remains: {}", getBufferedDataSize()); + return false; + } + + public boolean waitUntilFlusherTerminated(int maxWaitSeconds) throws InterruptedException { + int intervalMilli = 500; + for (int i = 0; i < maxWaitSeconds * (1000 / intervalMilli); i++) { + boolean terminated = isTerminated(); + LOG.debug("Waiting until the flusher is terminated: {}", terminated); + if (terminated) { + return true; + } + TimeUnit.MILLISECONDS.sleep(intervalMilli); + } + LOG.warn("The flusher isn't terminated"); + return false; + } + + public Buffer getBuffer() { + return buffer; + } + + public Flusher getFlusher() { + return flusher; + } + + @Override + public String toString() { + return "Fluency{" + "buffer=" + buffer + ", flusher=" + flusher + '}'; + } + + private interface Append { + void append() throws IOException; + } + + private class Emitter { + void emit(Append appender) throws IOException { + try { + appender.append(); + } catch (BufferFullException e) { + LOG.error("emit() failed due to buffer full. Flushing buffer. Please try again..."); flusher.flush(); + throw e; + } } - - @Override - public void close() - throws IOException - { - flusher.close(); - } - - public void clearBackupFiles() - { - buffer.clearBackupFiles(); - } - - public long getAllocatedBufferSize() - { - return buffer.getAllocatedSize(); - } - - public long getBufferedDataSize() - { - return buffer.getBufferedDataSize(); - } - - public boolean isTerminated() - { - return flusher.isTerminated(); - } - - public boolean waitUntilAllBufferFlushed(int maxWaitSeconds) - throws InterruptedException - { - int intervalMilli = 500; - for (int i = 0; i < maxWaitSeconds * (1000 / intervalMilli); i++) { - long bufferedDataSize = getBufferedDataSize(); - LOG.debug("Waiting for flushing all buffer: {}", bufferedDataSize); - if (bufferedDataSize == 0) { - return true; - } - TimeUnit.MILLISECONDS.sleep(intervalMilli); - } - LOG.warn("Buffered data still remains: {}", getBufferedDataSize()); - return false; - } - - public boolean waitUntilFlusherTerminated(int maxWaitSeconds) - throws InterruptedException - { - int intervalMilli = 500; - for (int i = 0; i < maxWaitSeconds * (1000 / intervalMilli); i++) { - boolean terminated = isTerminated(); - LOG.debug("Waiting until the flusher is terminated: {}", terminated); - if (terminated) { - return true; - } - TimeUnit.MILLISECONDS.sleep(intervalMilli); - } - LOG.warn("The flusher isn't terminated"); - return false; - } - - public Buffer getBuffer() - { - return buffer; - } - - public Flusher getFlusher() - { - return flusher; - } - - @Override - public String toString() - { - return "Fluency{" + - "buffer=" + buffer + - ", flusher=" + flusher + - '}'; - } - - private interface Append - { - void append() - throws IOException; - } - - private class Emitter - { - void emit(Append appender) - throws IOException - { - try { - appender.append(); - } - catch (BufferFullException e) { - LOG.error("emit() failed due to buffer full. Flushing buffer. Please try again..."); - flusher.flush(); - throw e; - } - } - } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/FluencyBuilder.java b/fluency-core/src/main/java/org/komamitsu/fluency/FluencyBuilder.java index 50179baf..f3a65fe6 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/FluencyBuilder.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/FluencyBuilder.java @@ -23,218 +23,202 @@ import org.komamitsu.fluency.recordformat.RecordFormatter; import org.msgpack.core.annotations.VisibleForTesting; -public class FluencyBuilder -{ - private Long maxBufferSize; - private Integer bufferChunkInitialSize; - private Integer bufferChunkRetentionSize; - private Integer bufferChunkRetentionTimeMillis; - private Integer flushAttemptIntervalMillis; - private String fileBackupDir; - private Integer waitUntilBufferFlushed; - private Integer waitUntilFlusherTerminated; - private Boolean jvmHeapBufferMode; - private ErrorHandler errorHandler; - - // TODO: Contain sender's retry setting here? - - public Long getMaxBufferSize() - { - return maxBufferSize; - } - - public void setMaxBufferSize(Long maxBufferSize) - { - this.maxBufferSize = maxBufferSize; - } - - public Integer getBufferChunkInitialSize() - { - return bufferChunkInitialSize; - } - - public void setBufferChunkInitialSize(Integer bufferChunkInitialSize) - { - this.bufferChunkInitialSize = bufferChunkInitialSize; - } - - public Integer getBufferChunkRetentionSize() - { - return bufferChunkRetentionSize; - } - - public void setBufferChunkRetentionSize(Integer bufferChunkRetentionSize) - { - this.bufferChunkRetentionSize = bufferChunkRetentionSize; - } - - public Integer getBufferChunkRetentionTimeMillis() - { - return bufferChunkRetentionTimeMillis; - } - - public void setBufferChunkRetentionTimeMillis(Integer bufferChunkRetentionTimeMillis) - { - this.bufferChunkRetentionTimeMillis = bufferChunkRetentionTimeMillis; - } - - /** - * @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} - */ - @Deprecated - public Integer getFlushIntervalMillis() - { - return flushAttemptIntervalMillis; - } - - /** - * @deprecated As of release 2.4.0, replaced by {@link #setFlushAttemptIntervalMillis(Integer flushAttemptIntervalMillis)} - */ - @Deprecated - public void setFlushIntervalMillis(Integer flushAttemptIntervalMillis) - { - this.flushAttemptIntervalMillis = flushAttemptIntervalMillis; - } - - public Integer getFlushAttemptIntervalMillis() - { - return flushAttemptIntervalMillis; - } +public class FluencyBuilder { + private Long maxBufferSize; + private Integer bufferChunkInitialSize; + private Integer bufferChunkRetentionSize; + private Integer bufferChunkRetentionTimeMillis; + private Integer flushAttemptIntervalMillis; + private String fileBackupDir; + private Integer waitUntilBufferFlushed; + private Integer waitUntilFlusherTerminated; + private Boolean jvmHeapBufferMode; + private ErrorHandler errorHandler; + + // TODO: Contain sender's retry setting here? + + public Long getMaxBufferSize() { + return maxBufferSize; + } + + public void setMaxBufferSize(Long maxBufferSize) { + this.maxBufferSize = maxBufferSize; + } + + public Integer getBufferChunkInitialSize() { + return bufferChunkInitialSize; + } + + public void setBufferChunkInitialSize(Integer bufferChunkInitialSize) { + this.bufferChunkInitialSize = bufferChunkInitialSize; + } + + public Integer getBufferChunkRetentionSize() { + return bufferChunkRetentionSize; + } + + public void setBufferChunkRetentionSize(Integer bufferChunkRetentionSize) { + this.bufferChunkRetentionSize = bufferChunkRetentionSize; + } + + public Integer getBufferChunkRetentionTimeMillis() { + return bufferChunkRetentionTimeMillis; + } + + public void setBufferChunkRetentionTimeMillis(Integer bufferChunkRetentionTimeMillis) { + this.bufferChunkRetentionTimeMillis = bufferChunkRetentionTimeMillis; + } + + /** + * @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} + */ + @Deprecated + public Integer getFlushIntervalMillis() { + return flushAttemptIntervalMillis; + } + + /** + * @deprecated As of release 2.4.0, replaced by {@link #setFlushAttemptIntervalMillis(Integer + * flushAttemptIntervalMillis)} + */ + @Deprecated + public void setFlushIntervalMillis(Integer flushAttemptIntervalMillis) { + this.flushAttemptIntervalMillis = flushAttemptIntervalMillis; + } + + public Integer getFlushAttemptIntervalMillis() { + return flushAttemptIntervalMillis; + } + + public void setFlushAttemptIntervalMillis(Integer flushAttemptIntervalMillis) { + this.flushAttemptIntervalMillis = flushAttemptIntervalMillis; + } + + public String getFileBackupDir() { + return fileBackupDir; + } + + public void setFileBackupDir(String fileBackupDir) { + this.fileBackupDir = fileBackupDir; + } + + public Integer getWaitUntilBufferFlushed() { + return waitUntilBufferFlushed; + } + + public void setWaitUntilBufferFlushed(Integer waitUntilBufferFlushed) { + this.waitUntilBufferFlushed = waitUntilBufferFlushed; + } + + public Integer getWaitUntilFlusherTerminated() { + return waitUntilFlusherTerminated; + } + + public void setWaitUntilFlusherTerminated(Integer waitUntilFlusherTerminated) { + this.waitUntilFlusherTerminated = waitUntilFlusherTerminated; + } + + public Boolean getJvmHeapBufferMode() { + return jvmHeapBufferMode; + } + + public void setJvmHeapBufferMode(Boolean jvmHeapBufferMode) { + this.jvmHeapBufferMode = jvmHeapBufferMode; + } + + public ErrorHandler getErrorHandler() { + return errorHandler; + } + + public void setErrorHandler(ErrorHandler errorHandler) { + this.errorHandler = errorHandler; + } + + @Override + public String toString() { + return "BaseFluencyBuilder{" + + "maxBufferSize=" + + maxBufferSize + + ", bufferChunkInitialSize=" + + bufferChunkInitialSize + + ", bufferChunkRetentionSize=" + + bufferChunkRetentionSize + + ", bufferChunkRetentionTimeMillis=" + + bufferChunkRetentionTimeMillis + + ", flushAttemptIntervalMillis=" + + flushAttemptIntervalMillis + + ", fileBackupDir='" + + fileBackupDir + + '\'' + + ", waitUntilBufferFlushed=" + + waitUntilBufferFlushed + + ", waitUntilFlusherTerminated=" + + waitUntilFlusherTerminated + + ", jvmHeapBufferMode=" + + jvmHeapBufferMode + + ", errorHandler=" + + errorHandler + + '}'; + } + + @VisibleForTesting + public Fluency createFluency( + RecordFormatter recordFormatter, + Ingester ingester, + Buffer.Config bufferConfig, + Flusher.Config flusherConfig) { + Buffer buffer = new Buffer(bufferConfig, recordFormatter); + Flusher flusher = new Flusher(flusherConfig, buffer, ingester); + return new Fluency(buffer, flusher); + } + + public Fluency buildFromIngester(RecordFormatter recordFormatter, Ingester ingester) { + Buffer.Config bufferConfig = new Buffer.Config(); + configureBufferConfig(bufferConfig); - public void setFlushAttemptIntervalMillis(Integer flushAttemptIntervalMillis) - { - this.flushAttemptIntervalMillis = flushAttemptIntervalMillis; - } - - public String getFileBackupDir() - { - return fileBackupDir; - } + Flusher.Config flusherConfig = new Flusher.Config(); + configureFlusherConfig(flusherConfig); - public void setFileBackupDir(String fileBackupDir) - { - this.fileBackupDir = fileBackupDir; - } - - public Integer getWaitUntilBufferFlushed() - { - return waitUntilBufferFlushed; - } + return createFluency(recordFormatter, ingester, bufferConfig, flusherConfig); + } - public void setWaitUntilBufferFlushed(Integer waitUntilBufferFlushed) - { - this.waitUntilBufferFlushed = waitUntilBufferFlushed; + private void configureBufferConfig(Buffer.Config bufferConfig) { + if (getMaxBufferSize() != null) { + bufferConfig.setMaxBufferSize(getMaxBufferSize()); } - public Integer getWaitUntilFlusherTerminated() - { - return waitUntilFlusherTerminated; + if (getBufferChunkInitialSize() != null) { + bufferConfig.setChunkInitialSize(getBufferChunkInitialSize()); } - public void setWaitUntilFlusherTerminated(Integer waitUntilFlusherTerminated) - { - this.waitUntilFlusherTerminated = waitUntilFlusherTerminated; + if (getBufferChunkRetentionSize() != null) { + bufferConfig.setChunkRetentionSize(getBufferChunkRetentionSize()); } - public Boolean getJvmHeapBufferMode() - { - return jvmHeapBufferMode; + if (getBufferChunkRetentionTimeMillis() != null) { + bufferConfig.setChunkRetentionTimeMillis(getBufferChunkRetentionTimeMillis()); } - public void setJvmHeapBufferMode(Boolean jvmHeapBufferMode) - { - this.jvmHeapBufferMode = jvmHeapBufferMode; + if (getFileBackupDir() != null) { + bufferConfig.setFileBackupDir(getFileBackupDir()); } - public ErrorHandler getErrorHandler() - { - return errorHandler; + if (getJvmHeapBufferMode() != null) { + bufferConfig.setJvmHeapBufferMode(getJvmHeapBufferMode()); } + } - public void setErrorHandler(ErrorHandler errorHandler) - { - this.errorHandler = errorHandler; + protected void configureFlusherConfig(Flusher.Config flusherConfig) { + if (getFlushAttemptIntervalMillis() != null) { + flusherConfig.setFlushAttemptIntervalMillis(getFlushAttemptIntervalMillis()); } - @Override - public String toString() - { - return "BaseFluencyBuilder{" + - "maxBufferSize=" + maxBufferSize + - ", bufferChunkInitialSize=" + bufferChunkInitialSize + - ", bufferChunkRetentionSize=" + bufferChunkRetentionSize + - ", bufferChunkRetentionTimeMillis=" + bufferChunkRetentionTimeMillis + - ", flushAttemptIntervalMillis=" + flushAttemptIntervalMillis + - ", fileBackupDir='" + fileBackupDir + '\'' + - ", waitUntilBufferFlushed=" + waitUntilBufferFlushed + - ", waitUntilFlusherTerminated=" + waitUntilFlusherTerminated + - ", jvmHeapBufferMode=" + jvmHeapBufferMode + - ", errorHandler=" + errorHandler + - '}'; + if (getWaitUntilBufferFlushed() != null) { + flusherConfig.setWaitUntilBufferFlushed(getWaitUntilBufferFlushed()); } - @VisibleForTesting - public Fluency createFluency( - RecordFormatter recordFormatter, - Ingester ingester, - Buffer.Config bufferConfig, - Flusher.Config flusherConfig) - { - Buffer buffer = new Buffer(bufferConfig, recordFormatter); - Flusher flusher = new Flusher(flusherConfig, buffer, ingester); - return new Fluency(buffer, flusher); - } - - public Fluency buildFromIngester(RecordFormatter recordFormatter, Ingester ingester) - { - Buffer.Config bufferConfig = new Buffer.Config(); - configureBufferConfig(bufferConfig); - - Flusher.Config flusherConfig = new Flusher.Config(); - configureFlusherConfig(flusherConfig); - - return createFluency(recordFormatter, ingester, bufferConfig, flusherConfig); - } - - private void configureBufferConfig(Buffer.Config bufferConfig) - { - if (getMaxBufferSize() != null) { - bufferConfig.setMaxBufferSize(getMaxBufferSize()); - } - - if (getBufferChunkInitialSize() != null) { - bufferConfig.setChunkInitialSize(getBufferChunkInitialSize()); - } - - if (getBufferChunkRetentionSize() != null) { - bufferConfig.setChunkRetentionSize(getBufferChunkRetentionSize()); - } - - if (getBufferChunkRetentionTimeMillis() != null) { - bufferConfig.setChunkRetentionTimeMillis(getBufferChunkRetentionTimeMillis()); - } - - if (getFileBackupDir() != null) { - bufferConfig.setFileBackupDir(getFileBackupDir()); - } - - if (getJvmHeapBufferMode() != null) { - bufferConfig.setJvmHeapBufferMode(getJvmHeapBufferMode()); - } - } - - protected void configureFlusherConfig(Flusher.Config flusherConfig) - { - if (getFlushAttemptIntervalMillis() != null) { - flusherConfig.setFlushAttemptIntervalMillis(getFlushAttemptIntervalMillis()); - } - - if (getWaitUntilBufferFlushed() != null) { - flusherConfig.setWaitUntilBufferFlushed(getWaitUntilBufferFlushed()); - } - - if (getWaitUntilFlusherTerminated() != null) { - flusherConfig.setWaitUntilTerminated(getWaitUntilFlusherTerminated()); - } + if (getWaitUntilFlusherTerminated() != null) { + flusherConfig.setWaitUntilTerminated(getWaitUntilFlusherTerminated()); } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/NonRetryableException.java b/fluency-core/src/main/java/org/komamitsu/fluency/NonRetryableException.java index 39d6b27c..03c776a4 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/NonRetryableException.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/NonRetryableException.java @@ -16,16 +16,12 @@ package org.komamitsu.fluency; -public class NonRetryableException - extends RuntimeException -{ - public NonRetryableException(String message) - { - super(message); - } +public class NonRetryableException extends RuntimeException { + public NonRetryableException(String message) { + super(message); + } - public NonRetryableException(String message, Throwable cause) - { - super(message, cause); - } + public NonRetryableException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/RetryableException.java b/fluency-core/src/main/java/org/komamitsu/fluency/RetryableException.java index 7616bd3b..31c9d854 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/RetryableException.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/RetryableException.java @@ -16,16 +16,12 @@ package org.komamitsu.fluency; -public class RetryableException - extends RuntimeException -{ - public RetryableException(String message) - { - super(message); - } +public class RetryableException extends RuntimeException { + public RetryableException(String message) { + super(message); + } - public RetryableException(String message, Throwable cause) - { - super(message, cause); - } + public RetryableException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/buffer/Buffer.java b/fluency-core/src/main/java/org/komamitsu/fluency/buffer/Buffer.java index b0335f67..b5a55ce7 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/buffer/Buffer.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/buffer/Buffer.java @@ -16,16 +16,6 @@ package org.komamitsu.fluency.buffer; -import org.komamitsu.fluency.BufferFullException; -import org.komamitsu.fluency.EventTime; -import org.komamitsu.fluency.ingester.Ingester; -import org.komamitsu.fluency.recordformat.RecordFormatter; -import org.komamitsu.fluency.validation.Validatable; -import org.komamitsu.fluency.validation.annotation.DecimalMin; -import org.komamitsu.fluency.validation.annotation.Min; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.File; import java.io.IOException; @@ -40,611 +30,562 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicLong; +import org.komamitsu.fluency.BufferFullException; +import org.komamitsu.fluency.EventTime; +import org.komamitsu.fluency.ingester.Ingester; +import org.komamitsu.fluency.recordformat.RecordFormatter; +import org.komamitsu.fluency.validation.Validatable; +import org.komamitsu.fluency.validation.annotation.DecimalMin; +import org.komamitsu.fluency.validation.annotation.Min; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class Buffer - implements Closeable -{ - private static final Logger LOG = LoggerFactory.getLogger(Buffer.class); - private final FileBackup fileBackup; - private final RecordFormatter recordFormatter; - private final Config config; - - private final Map retentionBuffers = new HashMap<>(); - private final LinkedBlockingQueue flushableBuffers = new LinkedBlockingQueue<>(); - private final Queue backupBuffers = new ConcurrentLinkedQueue<>(); - private final BufferPool bufferPool; - - public Buffer(RecordFormatter recordFormatter) - { - this(new Config(), recordFormatter); - } - - public Buffer(final Config config, RecordFormatter recordFormatter) - { - config.validateValues(); - this.config = config; - - if (config.getFileBackupDir() != null) { - fileBackup = new FileBackup(new File(config.getFileBackupDir()), this, config.getFileBackupPrefix()); - } - else { - fileBackup = null; - } - - this.recordFormatter = recordFormatter; - - bufferPool = new BufferPool( - config.getChunkInitialSize(), config.getMaxBufferSize(), config.jvmHeapBufferMode); - - init(); - } - - private void init() - { - if (fileBackup != null) { - for (FileBackup.SavedBuffer savedBuffer : fileBackup.getSavedFiles()) { - savedBuffer.open((params, channel) -> { - try { - LOG.info("Loading buffer: params={}, buffer.size={}", params, channel.size()); - } - catch (IOException e) { - LOG.error("Failed to access the backup file: params={}", params, e); - } - loadBufferFromFile(params, channel); - }); - } - } - } +public class Buffer implements Closeable { + private static final Logger LOG = LoggerFactory.getLogger(Buffer.class); + private final FileBackup fileBackup; + private final RecordFormatter recordFormatter; + private final Config config; + + private final Map retentionBuffers = new HashMap<>(); + private final LinkedBlockingQueue flushableBuffers = new LinkedBlockingQueue<>(); + private final Queue backupBuffers = new ConcurrentLinkedQueue<>(); + private final BufferPool bufferPool; + + public Buffer(RecordFormatter recordFormatter) { + this(new Config(), recordFormatter); + } + + public Buffer(final Config config, RecordFormatter recordFormatter) { + config.validateValues(); + this.config = config; + + if (config.getFileBackupDir() != null) { + fileBackup = + new FileBackup(new File(config.getFileBackupDir()), this, config.getFileBackupPrefix()); + } else { + fileBackup = null; + } + + this.recordFormatter = recordFormatter; + + bufferPool = + new BufferPool( + config.getChunkInitialSize(), config.getMaxBufferSize(), config.jvmHeapBufferMode); + + init(); + } + + private void init() { + if (fileBackup != null) { + for (FileBackup.SavedBuffer savedBuffer : fileBackup.getSavedFiles()) { + savedBuffer.open( + (params, channel) -> { + try { + LOG.info("Loading buffer: params={}, buffer.size={}", params, channel.size()); + } catch (IOException e) { + LOG.error("Failed to access the backup file: params={}", params, e); + } + loadBufferFromFile(params, channel); + }); + } + } + } + + protected void saveBuffer(List params, ByteBuffer buffer) { + if (fileBackup == null) { + return; + } + LOG.info("Saving buffer: params={}, buffer={}", params, buffer); + fileBackup.saveBuffer(params, buffer); + } + + public void flush(Ingester ingester, boolean force) throws IOException { + LOG.trace("flush(): force={}, bufferUsage={}", force, getBufferUsage()); + flushInternal(ingester, force); + } + + @Override + public void close() { + try { + LOG.debug("Saving all buffers if needed"); + saveAllBuffersToFile(); + } catch (Exception e) { + LOG.warn("Failed to save all buffers", e); + } + LOG.debug("Closing buffers"); + closeInternal(); + } + + private long getMaxSize() { + return config.getMaxBufferSize(); + } + + public float getBufferUsage() { + return (float) getAllocatedSize() / getMaxSize(); + } - protected void saveBuffer(List params, ByteBuffer buffer) - { - if (fileBackup == null) { - return; - } - LOG.info("Saving buffer: params={}, buffer={}", params, buffer); - fileBackup.saveBuffer(params, buffer); + public void clearBackupFiles() { + if (fileBackup != null) { + for (FileBackup.SavedBuffer buffer : fileBackup.getSavedFiles()) { + buffer.remove(); + } } - - public void flush(Ingester ingester, boolean force) - throws IOException - { - LOG.trace("flush(): force={}, bufferUsage={}", force, getBufferUsage()); - flushInternal(ingester, force); + } + + public long getMaxBufferSize() { + return config.getMaxBufferSize(); + } + + public String getFileBackupPrefix() { + return config.getFileBackupPrefix(); + } + + public String getFileBackupDir() { + return config.getFileBackupDir(); + } + + private RetentionBuffer prepareBuffer(String tag, int writeSize) throws BufferFullException { + RetentionBuffer retentionBuffer = retentionBuffers.get(tag); + if (retentionBuffer != null && retentionBuffer.getByteBuffer().remaining() > writeSize) { + return retentionBuffer; + } + + int existingDataSize = 0; + int newBufferChunkRetentionSize; + if (retentionBuffer == null) { + newBufferChunkRetentionSize = config.getChunkInitialSize(); + } else { + existingDataSize = retentionBuffer.getByteBuffer().position(); + newBufferChunkRetentionSize = + (int) (retentionBuffer.getByteBuffer().capacity() * config.getChunkExpandRatio()); + } + + while (newBufferChunkRetentionSize < (writeSize + existingDataSize)) { + newBufferChunkRetentionSize *= config.getChunkExpandRatio(); + } + + ByteBuffer acquiredBuffer = bufferPool.acquireBuffer(newBufferChunkRetentionSize); + if (acquiredBuffer == null) { + throw new BufferFullException( + "Buffer is full. config=" + config + ", bufferPool=" + bufferPool); + } + + RetentionBuffer newBuffer = new RetentionBuffer(acquiredBuffer, System.currentTimeMillis()); + if (retentionBuffer != null) { + ByteBuffer buf = retentionBuffer.getByteBuffer().duplicate(); + buf.flip(); + newBuffer.getByteBuffer().put(buf); + bufferPool.returnBuffer(retentionBuffer.getByteBuffer()); + } + LOG.trace("prepareBuffer(): allocate a new buffer. tag={}, buffer={}", tag, newBuffer); + + retentionBuffers.put(tag, newBuffer); + return newBuffer; + } + + private void loadDataToRetentionBuffers(String tag, ByteBuffer src) throws IOException { + synchronized (retentionBuffers) { + RetentionBuffer retentionBuffer = retentionBuffers.get(tag); + if (retentionBuffer != null) { + moveRetentionBufferIfNeeded(tag, retentionBuffer, src.remaining()); + } + RetentionBuffer buffer = prepareBuffer(tag, src.remaining()); + buffer.getByteBuffer().put(src); + } + } + + protected void loadBufferFromFile(List params, FileChannel channel) { + if (params.size() != 1) { + throw new IllegalArgumentException("The number of params should be 1: params=" + params); + } + String tag = params.get(0); + + try { + MappedByteBuffer src = channel.map(FileChannel.MapMode.PRIVATE, 0, channel.size()); + loadDataToRetentionBuffers(tag, src); + } catch (Exception e) { + LOG.error("Failed to load data to flushableBuffers: params={}, channel={}", params, channel); + } + } + + private void saveBuffer(TaggableBuffer buffer) { + saveBuffer(Collections.singletonList(buffer.getTag()), buffer.getByteBuffer()); + } + + protected void saveAllBuffersToFile() throws IOException { + moveRetentionBuffersToFlushable(true); // Just in case + + TaggableBuffer flushableBuffer; + while ((flushableBuffer = flushableBuffers.poll()) != null) { + saveBuffer(flushableBuffer); + } + while ((flushableBuffer = backupBuffers.poll()) != null) { + saveBuffer(flushableBuffer); + } + } + + private void appendMapInternal(String tag, Object timestamp, Map data) + throws IOException { + loadDataToRetentionBuffers(tag, ByteBuffer.wrap(recordFormatter.format(tag, timestamp, data))); + } + + private void appendMessagePackMapValueInternal( + String tag, Object timestamp, byte[] mapValue, int offset, int len) throws IOException { + loadDataToRetentionBuffers( + tag, + ByteBuffer.wrap( + recordFormatter.formatFromMessagePack(tag, timestamp, mapValue, offset, len))); + } + + private void appendMessagePackMapValueInternal(String tag, Object timestamp, ByteBuffer mapValue) + throws IOException { + loadDataToRetentionBuffers( + tag, ByteBuffer.wrap(recordFormatter.formatFromMessagePack(tag, timestamp, mapValue))); + } + + public void append(String tag, long timestamp, Map data) throws IOException { + appendMapInternal(tag, timestamp, data); + } + + public void append(String tag, EventTime timestamp, Map data) throws IOException { + appendMapInternal(tag, timestamp, data); + } + + public void appendMessagePackMapValue( + String tag, long timestamp, byte[] mapValue, int offset, int len) throws IOException { + appendMessagePackMapValueInternal(tag, timestamp, mapValue, offset, len); + } + + public void appendMessagePackMapValue( + String tag, EventTime timestamp, byte[] mapValue, int offset, int len) throws IOException { + appendMessagePackMapValueInternal(tag, timestamp, mapValue, offset, len); + } + + public void appendMessagePackMapValue(String tag, long timestamp, ByteBuffer mapValue) + throws IOException { + appendMessagePackMapValueInternal(tag, timestamp, mapValue); + } + + public void appendMessagePackMapValue(String tag, EventTime timestamp, ByteBuffer mapValue) + throws IOException { + appendMessagePackMapValueInternal(tag, timestamp, mapValue); + } + + private void moveRetentionBufferIfNeeded(String tag, RetentionBuffer buffer, int additionalSize) + throws IOException { + if (buffer.getByteBuffer().position() + additionalSize > config.getChunkRetentionSize()) { + moveRetentionBufferToFlushable(tag, buffer); + } + } + + private void moveRetentionBuffersToFlushable(boolean force) throws IOException { + long expiredThreshold = System.currentTimeMillis() - config.getChunkRetentionTimeMillis(); + + synchronized (retentionBuffers) { + for (Map.Entry entry : retentionBuffers.entrySet()) { + // it can be null because moveRetentionBufferToFlushable() can set null + if (entry.getValue() != null) { + if (force || entry.getValue().getCreatedTimeMillis() < expiredThreshold) { + moveRetentionBufferToFlushable(entry.getKey(), entry.getValue()); + } + } + } + } + } + + private void moveRetentionBufferToFlushable(String tag, RetentionBuffer buffer) + throws IOException { + try { + LOG.trace("moveRetentionBufferToFlushable(): tag={}, buffer={}", tag, buffer); + ByteBuffer buf = buffer.getByteBuffer().duplicate(); + buf.flip(); + flushableBuffers.put(new TaggableBuffer(tag, buf)); + retentionBuffers.put(tag, null); + } catch (InterruptedException e) { + throw new IOException("Failed to move retention buffer due to interruption", e); + } + } + + // TODO: Can't Buffer hold `intester` as an instance variable? + protected void flushInternal(Ingester ingester, boolean force) throws IOException { + moveRetentionBuffersToFlushable(force); + + TaggableBuffer flushableBuffer; + while (!Thread.currentThread().isInterrupted() + && (flushableBuffer = flushableBuffers.poll()) != null) { + boolean keepBuffer = false; + try { + LOG.trace( + "flushInternal(): bufferUsage={}, flushableBuffer={}", + getBufferUsage(), + flushableBuffer); + String tag = flushableBuffer.getTag(); + ByteBuffer dataBuffer = flushableBuffer.getByteBuffer(); + ingester.ingest(tag, dataBuffer); + } catch (IOException e) { + LOG.warn( + "Failed to send data. The data is going to be saved into the buffer again: data={}", + flushableBuffer); + keepBuffer = true; + throw e; + } finally { + if (keepBuffer) { + try { + flushableBuffers.put(flushableBuffer); + } catch (InterruptedException e1) { + LOG.warn( + "Failed to save the data into the buffer. Trying to save it in extra buffer: chunk={}", + flushableBuffer); + Thread.currentThread().interrupt(); + backupBuffers.add(flushableBuffer); + } + } else { + bufferPool.returnBuffer(flushableBuffer.getByteBuffer()); + } + } + } + } + + protected synchronized void closeInternal() { + retentionBuffers.clear(); + bufferPool.releaseBuffers(); + } + + public long getAllocatedSize() { + return bufferPool.getAllocatedSize(); + } + + public long getBufferedDataSize() { + long size = 0; + synchronized (retentionBuffers) { + for (Map.Entry buffer : retentionBuffers.entrySet()) { + if (buffer.getValue() != null && buffer.getValue().getByteBuffer() != null) { + size += buffer.getValue().getByteBuffer().position(); + } + } + } + for (TaggableBuffer buffer : flushableBuffers) { + if (buffer.getByteBuffer() != null) { + size += buffer.getByteBuffer().remaining(); + } + } + return size; + } + + public boolean getJvmHeapBufferMode() { + return bufferPool.getJvmHeapBufferMode(); + } + + public String bufferFormatType() { + // To keep backward compatibility + return "packed_forward"; + } + + public int getChunkInitialSize() { + return config.getChunkInitialSize(); + } + + public float getChunkExpandRatio() { + return config.getChunkExpandRatio(); + } + + public int getChunkRetentionSize() { + return config.getChunkRetentionSize(); + } + + public int getChunkRetentionTimeMillis() { + return config.getChunkRetentionTimeMillis(); + } + + @Override + public String toString() { + return "PackedForwardBuffer{" + + "retentionBuffers=" + + retentionBuffers + + ", flushableBuffers=" + + flushableBuffers + + ", backupBuffers=" + + backupBuffers + + ", bufferPool=" + + bufferPool + + ", config=" + + config + + "} " + + super.toString(); + } + + private static class RetentionBuffer { + private final AtomicLong createdTimeMillis; + private final ByteBuffer byteBuffer; + + RetentionBuffer(ByteBuffer byteBuffer, long currentTimeMillis) { + this.byteBuffer = byteBuffer; + this.createdTimeMillis = new AtomicLong(currentTimeMillis); + } + + long getCreatedTimeMillis() { + return createdTimeMillis.get(); + } + + ByteBuffer getByteBuffer() { + return byteBuffer; } @Override - public void close() - { - try { - LOG.debug("Saving all buffers if needed"); - saveAllBuffersToFile(); - } - catch (Exception e) { - LOG.warn("Failed to save all buffers", e); - } - LOG.debug("Closing buffers"); - closeInternal(); + public String toString() { + return "RetentionBuffer{" + + "createdTimeMillis=" + + createdTimeMillis + + ", byteBuffer=" + + byteBuffer + + '}'; } + } - private long getMaxSize() - { - return config.getMaxBufferSize(); - } + private static class TaggableBuffer { + private final String tag; + private final ByteBuffer byteBuffer; - public float getBufferUsage() - { - return (float) getAllocatedSize() / getMaxSize(); + public TaggableBuffer(String tag, ByteBuffer byteBuffer) { + this.tag = tag; + this.byteBuffer = byteBuffer; } - public void clearBackupFiles() - { - if (fileBackup != null) { - for (FileBackup.SavedBuffer buffer : fileBackup.getSavedFiles()) { - buffer.remove(); - } - } - } - - public long getMaxBufferSize() - { - return config.getMaxBufferSize(); + public String getTag() { + return tag; } - public String getFileBackupPrefix() - { - return config.getFileBackupPrefix(); + public ByteBuffer getByteBuffer() { + return byteBuffer; } - public String getFileBackupDir() - { - return config.getFileBackupDir(); - } - - private RetentionBuffer prepareBuffer(String tag, int writeSize) - throws BufferFullException - { - RetentionBuffer retentionBuffer = retentionBuffers.get(tag); - if (retentionBuffer != null && retentionBuffer.getByteBuffer().remaining() > writeSize) { - return retentionBuffer; - } - - int existingDataSize = 0; - int newBufferChunkRetentionSize; - if (retentionBuffer == null) { - newBufferChunkRetentionSize = config.getChunkInitialSize(); - } - else { - existingDataSize = retentionBuffer.getByteBuffer().position(); - newBufferChunkRetentionSize = (int) (retentionBuffer.getByteBuffer().capacity() * config.getChunkExpandRatio()); - } - - while (newBufferChunkRetentionSize < (writeSize + existingDataSize)) { - newBufferChunkRetentionSize *= config.getChunkExpandRatio(); - } - - ByteBuffer acquiredBuffer = bufferPool.acquireBuffer(newBufferChunkRetentionSize); - if (acquiredBuffer == null) { - throw new BufferFullException("Buffer is full. config=" + config + ", bufferPool=" + bufferPool); - } - - RetentionBuffer newBuffer = new RetentionBuffer(acquiredBuffer, System.currentTimeMillis()); - if (retentionBuffer != null) { - ByteBuffer buf = retentionBuffer.getByteBuffer().duplicate(); - buf.flip(); - newBuffer.getByteBuffer().put(buf); - bufferPool.returnBuffer(retentionBuffer.getByteBuffer()); - } - LOG.trace("prepareBuffer(): allocate a new buffer. tag={}, buffer={}", tag, newBuffer); - - retentionBuffers.put(tag, newBuffer); - return newBuffer; - } - - private void loadDataToRetentionBuffers(String tag, ByteBuffer src) - throws IOException - { - synchronized (retentionBuffers) { - RetentionBuffer retentionBuffer = retentionBuffers.get(tag); - if (retentionBuffer != null) { - moveRetentionBufferIfNeeded(tag, retentionBuffer, src.remaining()); - } - RetentionBuffer buffer = prepareBuffer(tag, src.remaining()); - buffer.getByteBuffer().put(src); - } - } - - protected void loadBufferFromFile(List params, FileChannel channel) - { - if (params.size() != 1) { - throw new IllegalArgumentException("The number of params should be 1: params=" + params); - } - String tag = params.get(0); - - try { - MappedByteBuffer src = channel.map(FileChannel.MapMode.PRIVATE, 0, channel.size()); - loadDataToRetentionBuffers(tag, src); - } - catch (Exception e) { - LOG.error("Failed to load data to flushableBuffers: params={}, channel={}", params, channel); - } + @Override + public String toString() { + return "TaggableBuffer{" + "tag='" + tag + '\'' + ", byteBuffer=" + byteBuffer + '}'; } + } - private void saveBuffer(TaggableBuffer buffer) - { - saveBuffer(Collections.singletonList(buffer.getTag()), buffer.getByteBuffer()); - } + public static class Config implements Validatable { + private long maxBufferSize = 512 * 1024 * 1024; + private String fileBackupDir; + private String fileBackupPrefix; // Mainly for testing - protected void saveAllBuffersToFile() - throws IOException - { - moveRetentionBuffersToFlushable(true); // Just in case + private int chunkInitialSize = 1024 * 1024; - TaggableBuffer flushableBuffer; - while ((flushableBuffer = flushableBuffers.poll()) != null) { - saveBuffer(flushableBuffer); - } - while ((flushableBuffer = backupBuffers.poll()) != null) { - saveBuffer(flushableBuffer); - } - } + @DecimalMin("1.2") + private float chunkExpandRatio = 2.0f; - private void appendMapInternal(String tag, Object timestamp, Map data) - throws IOException - { - loadDataToRetentionBuffers(tag, - ByteBuffer.wrap(recordFormatter.format(tag, timestamp, data))); - } + private int chunkRetentionSize = 4 * 1024 * 1024; - private void appendMessagePackMapValueInternal(String tag, Object timestamp, byte[] mapValue, int offset, int len) - throws IOException - { - loadDataToRetentionBuffers(tag, - ByteBuffer.wrap(recordFormatter.formatFromMessagePack(tag, timestamp, mapValue, offset, len))); - } + @Min(50) + private int chunkRetentionTimeMillis = 1000; - private void appendMessagePackMapValueInternal(String tag, Object timestamp, ByteBuffer mapValue) - throws IOException - { - loadDataToRetentionBuffers(tag, - ByteBuffer.wrap(recordFormatter.formatFromMessagePack(tag, timestamp, mapValue))); - } + private boolean jvmHeapBufferMode = false; - public void append(String tag, long timestamp, Map data) - throws IOException - { - appendMapInternal(tag, timestamp, data); + public long getMaxBufferSize() { + return maxBufferSize; } - public void append(String tag, EventTime timestamp, Map data) - throws IOException - { - appendMapInternal(tag, timestamp, data); + public void setMaxBufferSize(long maxBufferSize) { + this.maxBufferSize = maxBufferSize; } - public void appendMessagePackMapValue(String tag, long timestamp, byte[] mapValue, int offset, int len) - throws IOException - { - appendMessagePackMapValueInternal(tag, timestamp, mapValue, offset, len); + public String getFileBackupDir() { + return fileBackupDir; } - public void appendMessagePackMapValue(String tag, EventTime timestamp, byte[] mapValue, int offset, int len) - throws IOException - { - appendMessagePackMapValueInternal(tag, timestamp, mapValue, offset, len); + public void setFileBackupDir(String fileBackupDir) { + this.fileBackupDir = fileBackupDir; } - public void appendMessagePackMapValue(String tag, long timestamp, ByteBuffer mapValue) - throws IOException - { - appendMessagePackMapValueInternal(tag, timestamp, mapValue); + public String getFileBackupPrefix() { + return fileBackupPrefix; } - public void appendMessagePackMapValue(String tag, EventTime timestamp, ByteBuffer mapValue) - throws IOException - { - appendMessagePackMapValueInternal(tag, timestamp, mapValue); + public void setFileBackupPrefix(String fileBackupPrefix) { + this.fileBackupPrefix = fileBackupPrefix; } - private void moveRetentionBufferIfNeeded(String tag, RetentionBuffer buffer, int additionalSize) - throws IOException - { - if (buffer.getByteBuffer().position() + additionalSize > config.getChunkRetentionSize()) { - moveRetentionBufferToFlushable(tag, buffer); - } + public int getChunkInitialSize() { + return chunkInitialSize; } - private void moveRetentionBuffersToFlushable(boolean force) - throws IOException - { - long expiredThreshold = System.currentTimeMillis() - config.getChunkRetentionTimeMillis(); - - synchronized (retentionBuffers) { - for (Map.Entry entry : retentionBuffers.entrySet()) { - // it can be null because moveRetentionBufferToFlushable() can set null - if (entry.getValue() != null) { - if (force || entry.getValue().getCreatedTimeMillis() < expiredThreshold) { - moveRetentionBufferToFlushable(entry.getKey(), entry.getValue()); - } - } - } - } + public void setChunkInitialSize(int chunkInitialSize) { + this.chunkInitialSize = chunkInitialSize; } - private void moveRetentionBufferToFlushable(String tag, RetentionBuffer buffer) - throws IOException - { - try { - LOG.trace("moveRetentionBufferToFlushable(): tag={}, buffer={}", tag, buffer); - ByteBuffer buf = buffer.getByteBuffer().duplicate(); - buf.flip(); - flushableBuffers.put(new TaggableBuffer(tag, buf)); - retentionBuffers.put(tag, null); - } - catch (InterruptedException e) { - throw new IOException("Failed to move retention buffer due to interruption", e); - } + public float getChunkExpandRatio() { + return chunkExpandRatio; } - // TODO: Can't Buffer hold `intester` as an instance variable? - protected void flushInternal(Ingester ingester, boolean force) - throws IOException - { - moveRetentionBuffersToFlushable(force); - - TaggableBuffer flushableBuffer; - while (!Thread.currentThread().isInterrupted() && - (flushableBuffer = flushableBuffers.poll()) != null) { - boolean keepBuffer = false; - try { - LOG.trace("flushInternal(): bufferUsage={}, flushableBuffer={}", getBufferUsage(), flushableBuffer); - String tag = flushableBuffer.getTag(); - ByteBuffer dataBuffer = flushableBuffer.getByteBuffer(); - ingester.ingest(tag, dataBuffer); - } - catch (IOException e) { - LOG.warn("Failed to send data. The data is going to be saved into the buffer again: data={}", flushableBuffer); - keepBuffer = true; - throw e; - } - finally { - if (keepBuffer) { - try { - flushableBuffers.put(flushableBuffer); - } - catch (InterruptedException e1) { - LOG.warn("Failed to save the data into the buffer. Trying to save it in extra buffer: chunk={}", flushableBuffer); - Thread.currentThread().interrupt(); - backupBuffers.add(flushableBuffer); - } - } - else { - bufferPool.returnBuffer(flushableBuffer.getByteBuffer()); - } - } - } + public void setChunkExpandRatio(float chunkExpandRatio) { + this.chunkExpandRatio = chunkExpandRatio; } - protected synchronized void closeInternal() - { - retentionBuffers.clear(); - bufferPool.releaseBuffers(); + public int getChunkRetentionSize() { + return chunkRetentionSize; } - public long getAllocatedSize() - { - return bufferPool.getAllocatedSize(); + public void setChunkRetentionSize(int chunkRetentionSize) { + this.chunkRetentionSize = chunkRetentionSize; } - public long getBufferedDataSize() - { - long size = 0; - synchronized (retentionBuffers) { - for (Map.Entry buffer : retentionBuffers.entrySet()) { - if (buffer.getValue() != null && buffer.getValue().getByteBuffer() != null) { - size += buffer.getValue().getByteBuffer().position(); - } - } - } - for (TaggableBuffer buffer : flushableBuffers) { - if (buffer.getByteBuffer() != null) { - size += buffer.getByteBuffer().remaining(); - } - } - return size; + public int getChunkRetentionTimeMillis() { + return chunkRetentionTimeMillis; } - public boolean getJvmHeapBufferMode() - { - return bufferPool.getJvmHeapBufferMode(); + public void setChunkRetentionTimeMillis(int chunkRetentionTimeMillis) { + this.chunkRetentionTimeMillis = chunkRetentionTimeMillis; } - public String bufferFormatType() - { - // To keep backward compatibility - return "packed_forward"; + public boolean getJvmHeapBufferMode() { + return jvmHeapBufferMode; } - public int getChunkInitialSize() - { - return config.getChunkInitialSize(); + public void setJvmHeapBufferMode(boolean jvmHeapBufferMode) { + this.jvmHeapBufferMode = jvmHeapBufferMode; } - public float getChunkExpandRatio() - { - return config.getChunkExpandRatio(); - } + void validateValues() { + validate(); - public int getChunkRetentionSize() - { - return config.getChunkRetentionSize(); - } + if (chunkInitialSize >= chunkRetentionSize) { + throw new IllegalArgumentException( + String.format( + "Buffer Chunk Retention Size (%d) should be more than Initial Buffer Chunk Size (%d)", + chunkRetentionSize, chunkInitialSize)); + } - public int getChunkRetentionTimeMillis() - { - return config.getChunkRetentionTimeMillis(); + if (chunkRetentionSize >= maxBufferSize) { + throw new IllegalArgumentException( + String.format( + "Max Total Buffer Size (%d) should be more than Buffer Chunk Retention Size (%d)", + maxBufferSize, chunkRetentionSize)); + } } @Override - public String toString() - { - return "PackedForwardBuffer{" + - "retentionBuffers=" + retentionBuffers + - ", flushableBuffers=" + flushableBuffers + - ", backupBuffers=" + backupBuffers + - ", bufferPool=" + bufferPool + - ", config=" + config + - "} " + super.toString(); - } - - private static class RetentionBuffer - { - private final AtomicLong createdTimeMillis; - private final ByteBuffer byteBuffer; - - RetentionBuffer(ByteBuffer byteBuffer, long currentTimeMillis) - { - this.byteBuffer = byteBuffer; - this.createdTimeMillis = new AtomicLong(currentTimeMillis); - } - - long getCreatedTimeMillis() - { - return createdTimeMillis.get(); - } - - ByteBuffer getByteBuffer() - { - return byteBuffer; - } - - @Override - public String toString() - { - return "RetentionBuffer{" + - "createdTimeMillis=" + createdTimeMillis + - ", byteBuffer=" + byteBuffer + - '}'; - } - } - - private static class TaggableBuffer - { - private final String tag; - private final ByteBuffer byteBuffer; - - public TaggableBuffer(String tag, ByteBuffer byteBuffer) - { - this.tag = tag; - this.byteBuffer = byteBuffer; - } - - public String getTag() - { - return tag; - } - - public ByteBuffer getByteBuffer() - { - return byteBuffer; - } - - @Override - public String toString() - { - return "TaggableBuffer{" + - "tag='" + tag + '\'' + - ", byteBuffer=" + byteBuffer + - '}'; - } - } - - public static class Config - implements Validatable - { - private long maxBufferSize = 512 * 1024 * 1024; - private String fileBackupDir; - private String fileBackupPrefix; // Mainly for testing - - private int chunkInitialSize = 1024 * 1024; - @DecimalMin("1.2") - private float chunkExpandRatio = 2.0f; - private int chunkRetentionSize = 4 * 1024 * 1024; - @Min(50) - private int chunkRetentionTimeMillis = 1000; - private boolean jvmHeapBufferMode = false; - - public long getMaxBufferSize() - { - return maxBufferSize; - } - - public void setMaxBufferSize(long maxBufferSize) - { - this.maxBufferSize = maxBufferSize; - } - - public String getFileBackupDir() - { - return fileBackupDir; - } - - public void setFileBackupDir(String fileBackupDir) - { - this.fileBackupDir = fileBackupDir; - } - - public String getFileBackupPrefix() - { - return fileBackupPrefix; - } - - public void setFileBackupPrefix(String fileBackupPrefix) - { - this.fileBackupPrefix = fileBackupPrefix; - } - - public int getChunkInitialSize() - { - return chunkInitialSize; - } - - public void setChunkInitialSize(int chunkInitialSize) - { - this.chunkInitialSize = chunkInitialSize; - } - - public float getChunkExpandRatio() - { - return chunkExpandRatio; - } - - public void setChunkExpandRatio(float chunkExpandRatio) - { - this.chunkExpandRatio = chunkExpandRatio; - } - - public int getChunkRetentionSize() - { - return chunkRetentionSize; - } - - public void setChunkRetentionSize(int chunkRetentionSize) - { - this.chunkRetentionSize = chunkRetentionSize; - } - - public int getChunkRetentionTimeMillis() - { - return chunkRetentionTimeMillis; - } - - public void setChunkRetentionTimeMillis(int chunkRetentionTimeMillis) - { - this.chunkRetentionTimeMillis = chunkRetentionTimeMillis; - } - - public boolean getJvmHeapBufferMode() - { - return jvmHeapBufferMode; - } - - public void setJvmHeapBufferMode(boolean jvmHeapBufferMode) - { - this.jvmHeapBufferMode = jvmHeapBufferMode; - } - - void validateValues() - { - validate(); - - if (chunkInitialSize >= chunkRetentionSize) { - throw new IllegalArgumentException( - String.format( - "Buffer Chunk Retention Size (%d) should be more than Initial Buffer Chunk Size (%d)", - chunkRetentionSize, chunkInitialSize)); - } - - if (chunkRetentionSize >= maxBufferSize) { - throw new IllegalArgumentException( - String.format( - "Max Total Buffer Size (%d) should be more than Buffer Chunk Retention Size (%d)", - maxBufferSize, chunkRetentionSize)); - } - } - - @Override - public String toString() - { - return "Config{" + - "maxBufferSize=" + maxBufferSize + - ", fileBackupDir='" + fileBackupDir + '\'' + - ", fileBackupPrefix='" + fileBackupPrefix + '\'' + - ", chunkInitialSize=" + chunkInitialSize + - ", chunkExpandRatio=" + chunkExpandRatio + - ", chunkRetentionSize=" + chunkRetentionSize + - ", chunkRetentionTimeMillis=" + chunkRetentionTimeMillis + - ", jvmHeapBufferMode=" + jvmHeapBufferMode + - '}'; - } - } + public String toString() { + return "Config{" + + "maxBufferSize=" + + maxBufferSize + + ", fileBackupDir='" + + fileBackupDir + + '\'' + + ", fileBackupPrefix='" + + fileBackupPrefix + + '\'' + + ", chunkInitialSize=" + + chunkInitialSize + + ", chunkExpandRatio=" + + chunkExpandRatio + + ", chunkRetentionSize=" + + chunkRetentionSize + + ", chunkRetentionTimeMillis=" + + chunkRetentionTimeMillis + + ", jvmHeapBufferMode=" + + jvmHeapBufferMode + + '}'; + } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/buffer/BufferPool.java b/fluency-core/src/main/java/org/komamitsu/fluency/buffer/BufferPool.java index 5e1d2c2a..187f142d 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/buffer/BufferPool.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/buffer/BufferPool.java @@ -16,129 +16,125 @@ package org.komamitsu.fluency.buffer; -import org.msgpack.core.annotations.VisibleForTesting; - import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicLong; +import org.msgpack.core.annotations.VisibleForTesting; -class BufferPool -{ - @VisibleForTesting final Map> bufferPool = new HashMap<>(); - private final AtomicLong allocatedSize = new AtomicLong(); - private final int initialBufferSize; - private final long maxBufferSize; - private final boolean jvmHeapBufferMode; - - public BufferPool(int initialBufferSize, long maxBufferSize) - { - this(initialBufferSize, maxBufferSize, false); +class BufferPool { + @VisibleForTesting + final Map> bufferPool = new HashMap<>(); + + private final AtomicLong allocatedSize = new AtomicLong(); + private final int initialBufferSize; + private final long maxBufferSize; + private final boolean jvmHeapBufferMode; + + public BufferPool(int initialBufferSize, long maxBufferSize) { + this(initialBufferSize, maxBufferSize, false); + } + + public BufferPool(int initialBufferSize, long maxBufferSize, boolean jvmHeapBufferMode) { + this.initialBufferSize = initialBufferSize; + this.maxBufferSize = maxBufferSize; + this.jvmHeapBufferMode = jvmHeapBufferMode; + } + + public ByteBuffer acquireBuffer(int bufferSize) { + int normalizedBufferSize = initialBufferSize; + while (normalizedBufferSize < bufferSize) { + normalizedBufferSize *= 2; } - public BufferPool(int initialBufferSize, long maxBufferSize, boolean jvmHeapBufferMode) - { - this.initialBufferSize = initialBufferSize; - this.maxBufferSize = maxBufferSize; - this.jvmHeapBufferMode = jvmHeapBufferMode; + LinkedBlockingQueue buffers; + synchronized (bufferPool) { + buffers = bufferPool.get(normalizedBufferSize); + if (buffers == null) { + buffers = new LinkedBlockingQueue<>(); + bufferPool.put(normalizedBufferSize, buffers); + } } - public ByteBuffer acquireBuffer(int bufferSize) - { - int normalizedBufferSize = initialBufferSize; - while (normalizedBufferSize < bufferSize) { - normalizedBufferSize *= 2; - } - - LinkedBlockingQueue buffers; - synchronized (bufferPool) { - buffers = bufferPool.get(normalizedBufferSize); - if (buffers == null) { - buffers = new LinkedBlockingQueue<>(); - bufferPool.put(normalizedBufferSize, buffers); - } - } - - ByteBuffer buffer = buffers.poll(); - if (buffer != null) { - return buffer; - } - - /* - synchronized (allocatedSize) { - if (allocatedSize.get() + normalizedBufferSize > maxBufferSize) { - return null; // `null` means the buffer is full. - } - allocatedSize.addAndGet(normalizedBufferSize); - return ByteBuffer.allocateDirect(normalizedBufferSize); - } - */ - - while (true) { - long currentAllocatedSize = allocatedSize.get(); - if (currentAllocatedSize + normalizedBufferSize > maxBufferSize) { - releaseBuffers(); - return null; // `null` means the buffer is full. - } - if (currentAllocatedSize == allocatedSize.getAndAdd(normalizedBufferSize)) { - ByteBuffer buf; - if (jvmHeapBufferMode) { - buf = ByteBuffer.allocate(normalizedBufferSize); - } - else { - buf = ByteBuffer.allocateDirect(normalizedBufferSize); - } - return buf; - } - allocatedSize.getAndAdd(-normalizedBufferSize); - } + ByteBuffer buffer = buffers.poll(); + if (buffer != null) { + return buffer; } - public void returnBuffer(ByteBuffer byteBuffer) - { - LinkedBlockingQueue buffers = bufferPool.get(byteBuffer.capacity()); - if (buffers == null) { - throw new IllegalStateException("`buffers` shouldn't be null"); + /* + synchronized (allocatedSize) { + if (allocatedSize.get() + normalizedBufferSize > maxBufferSize) { + return null; // `null` means the buffer is full. } - - byteBuffer.position(0); - byteBuffer.limit(byteBuffer.capacity()); - buffers.offer(byteBuffer); + allocatedSize.addAndGet(normalizedBufferSize); + return ByteBuffer.allocateDirect(normalizedBufferSize); } - - public long getAllocatedSize() - { - return allocatedSize.get(); - } - - public void releaseBuffers() - { - // TODO: Stop releasing all the buffers when having released buffers to some extent - synchronized (bufferPool) { - for (Map.Entry> entry : bufferPool.entrySet()) { - ByteBuffer buffer; - while ((buffer = entry.getValue().poll()) != null) { - allocatedSize.addAndGet(-buffer.capacity()); - } - } + */ + + while (true) { + long currentAllocatedSize = allocatedSize.get(); + if (currentAllocatedSize + normalizedBufferSize > maxBufferSize) { + releaseBuffers(); + return null; // `null` means the buffer is full. + } + if (currentAllocatedSize == allocatedSize.getAndAdd(normalizedBufferSize)) { + ByteBuffer buf; + if (jvmHeapBufferMode) { + buf = ByteBuffer.allocate(normalizedBufferSize); + } else { + buf = ByteBuffer.allocateDirect(normalizedBufferSize); } + return buf; + } + allocatedSize.getAndAdd(-normalizedBufferSize); } + } - public boolean getJvmHeapBufferMode() - { - return jvmHeapBufferMode; + public void returnBuffer(ByteBuffer byteBuffer) { + LinkedBlockingQueue buffers = bufferPool.get(byteBuffer.capacity()); + if (buffers == null) { + throw new IllegalStateException("`buffers` shouldn't be null"); } - @Override - public String toString() - { - return "BufferPool{" + - "bufferPool=" + bufferPool + - ", allocatedSize=" + allocatedSize + - ", initialBufferSize=" + initialBufferSize + - ", maxBufferSize=" + maxBufferSize + - ", jvmHeapBufferMode=" + jvmHeapBufferMode + - '}'; + byteBuffer.position(0); + byteBuffer.limit(byteBuffer.capacity()); + buffers.offer(byteBuffer); + } + + public long getAllocatedSize() { + return allocatedSize.get(); + } + + public void releaseBuffers() { + // TODO: Stop releasing all the buffers when having released buffers to some extent + synchronized (bufferPool) { + for (Map.Entry> entry : bufferPool.entrySet()) { + ByteBuffer buffer; + while ((buffer = entry.getValue().poll()) != null) { + allocatedSize.addAndGet(-buffer.capacity()); + } + } } + } + + public boolean getJvmHeapBufferMode() { + return jvmHeapBufferMode; + } + + @Override + public String toString() { + return "BufferPool{" + + "bufferPool=" + + bufferPool + + ", allocatedSize=" + + allocatedSize + + ", initialBufferSize=" + + initialBufferSize + + ", maxBufferSize=" + + maxBufferSize + + ", jvmHeapBufferMode=" + + jvmHeapBufferMode + + '}'; + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/buffer/FileBackup.java b/fluency-core/src/main/java/org/komamitsu/fluency/buffer/FileBackup.java index 257aca4c..c7b9c730 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/buffer/FileBackup.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/buffer/FileBackup.java @@ -16,9 +16,6 @@ package org.komamitsu.fluency.buffer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.File; import java.io.FileOutputStream; @@ -33,199 +30,198 @@ import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class FileBackup -{ - private static final Logger LOG = LoggerFactory.getLogger(FileBackup.class); - private static final String PARAM_DELIM_IN_FILENAME = "#"; - private static final String EXT_FILENAME = ".buf"; - private final File backupDir; - private final Buffer userBuffer; - private final Pattern pattern; - private final String prefix; - - public FileBackup(File backupDir, Buffer userBuffer, String prefix) - { - if (backupDir.mkdir()) { - LOG.info("Created backupDir: dir={}", backupDir); - } - if (!backupDir.isDirectory() || !backupDir.canRead() || !backupDir.canWrite()) { - throw new IllegalArgumentException("backupDir[" + backupDir + "] needs to be a readable & writable directory"); - } - this.backupDir = backupDir; - this.userBuffer = userBuffer; - this.prefix = prefix; - this.pattern = Pattern.compile(String.format("^%s%s%s([\\w.\\-%s]+)%s$", userBuffer.bufferFormatType(), prefix(), PARAM_DELIM_IN_FILENAME, PARAM_DELIM_IN_FILENAME, EXT_FILENAME)); - LOG.debug(this.toString()); +public class FileBackup { + private static final Logger LOG = LoggerFactory.getLogger(FileBackup.class); + private static final String PARAM_DELIM_IN_FILENAME = "#"; + private static final String EXT_FILENAME = ".buf"; + private final File backupDir; + private final Buffer userBuffer; + private final Pattern pattern; + private final String prefix; + + public FileBackup(File backupDir, Buffer userBuffer, String prefix) { + if (backupDir.mkdir()) { + LOG.info("Created backupDir: dir={}", backupDir); } - - private String prefix() - { - return prefix == null ? "" : "_" + prefix; + if (!backupDir.isDirectory() || !backupDir.canRead() || !backupDir.canWrite()) { + throw new IllegalArgumentException( + "backupDir[" + backupDir + "] needs to be a readable & writable directory"); } - - @Override - public String toString() - { - return "FileBackup{" + - "backupDir=" + backupDir + - ", userBuffer=" + userBuffer + - ", pattern=" + pattern + - ", prefix='" + prefix + '\'' + - '}'; + this.backupDir = backupDir; + this.userBuffer = userBuffer; + this.prefix = prefix; + this.pattern = + Pattern.compile( + String.format( + "^%s%s%s([\\w.\\-%s]+)%s$", + userBuffer.bufferFormatType(), + prefix(), + PARAM_DELIM_IN_FILENAME, + PARAM_DELIM_IN_FILENAME, + EXT_FILENAME)); + LOG.debug(this.toString()); + } + + private String prefix() { + return prefix == null ? "" : "_" + prefix; + } + + @Override + public String toString() { + return "FileBackup{" + + "backupDir=" + + backupDir + + ", userBuffer=" + + userBuffer + + ", pattern=" + + pattern + + ", prefix='" + + prefix + + '\'' + + '}'; + } + + public List getSavedFiles() { + File[] files = backupDir.listFiles(); + if (files == null) { + LOG.warn("Failed to list the backup directory. {}", backupDir); + return new ArrayList<>(); } - public List getSavedFiles() - { - File[] files = backupDir.listFiles(); - if (files == null) { - LOG.warn("Failed to list the backup directory. {}", backupDir); - return new ArrayList<>(); - } - - LOG.debug("Checking backup files. files.length={}", files.length); - List savedBuffers = new ArrayList<>(); - for (File f : files) { - Matcher matcher = pattern.matcher(f.toPath().getFileName().toString()); - if (matcher.find()) { - if (matcher.groupCount() != 1) { - LOG.warn("Invalid backup filename: file={}", f.getName()); - } - else { - String concatParams = matcher.group(1); - String[] params = concatParams.split(PARAM_DELIM_IN_FILENAME); - LinkedList paramList = new LinkedList<>(Arrays.asList(params)); - LOG.debug("Saved buffer params={}", paramList); - paramList.removeLast(); - savedBuffers.add(new SavedBuffer(f, paramList)); - } - } - else { - LOG.trace("Found a file in backup dir, but the file path doesn't match the pattern. file={}", f.getAbsolutePath()); - } + LOG.debug("Checking backup files. files.length={}", files.length); + List savedBuffers = new ArrayList<>(); + for (File f : files) { + Matcher matcher = pattern.matcher(f.toPath().getFileName().toString()); + if (matcher.find()) { + if (matcher.groupCount() != 1) { + LOG.warn("Invalid backup filename: file={}", f.getName()); + } else { + String concatParams = matcher.group(1); + String[] params = concatParams.split(PARAM_DELIM_IN_FILENAME); + LinkedList paramList = new LinkedList<>(Arrays.asList(params)); + LOG.debug("Saved buffer params={}", paramList); + paramList.removeLast(); + savedBuffers.add(new SavedBuffer(f, paramList)); + } + } else { + LOG.trace( + "Found a file in backup dir, but the file path doesn't match the pattern. file={}", + f.getAbsolutePath()); + } + } + return savedBuffers; + } + + public void saveBuffer(List params, ByteBuffer buffer) { + List copiedParams = new ArrayList<>(params); + copiedParams.add(String.valueOf(System.nanoTime())); + + boolean isFirst = true; + StringBuilder sb = new StringBuilder(); + for (String param : copiedParams) { + if (isFirst) { + isFirst = false; + } else { + sb.append(PARAM_DELIM_IN_FILENAME); + } + sb.append(param); + } + String filename = + this.userBuffer.bufferFormatType() + + prefix() + + PARAM_DELIM_IN_FILENAME + + sb.toString() + + EXT_FILENAME; + + File file = new File(backupDir, filename); + LOG.debug("Backing up buffer: path={}, size={}", file.getAbsolutePath(), buffer.remaining()); + + FileChannel channel = null; + try { + channel = new FileOutputStream(file).getChannel(); + channel.write(buffer); + } catch (Exception e) { + LOG.error( + "Failed to save buffer to file: params={}, path={}, buffer={}", + copiedParams, + file.getAbsolutePath(), + buffer, + e); + } finally { + if (channel != null) { + try { + channel.close(); + } catch (IOException e) { + LOG.warn("Failed to close Channel: channel={}", channel); } - return savedBuffers; + } } + } - public void saveBuffer(List params, ByteBuffer buffer) - { - List copiedParams = new ArrayList<>(params); - copiedParams.add(String.valueOf(System.nanoTime())); - - boolean isFirst = true; - StringBuilder sb = new StringBuilder(); - for (String param : copiedParams) { - if (isFirst) { - isFirst = false; - } - else { - sb.append(PARAM_DELIM_IN_FILENAME); - } - sb.append(param); - } - String filename = this.userBuffer.bufferFormatType() + prefix() + PARAM_DELIM_IN_FILENAME + sb.toString() + EXT_FILENAME; + public static class SavedBuffer implements Closeable { + private final List params; + private final File savedFile; + private FileChannel channel; - File file = new File(backupDir, filename); - LOG.debug("Backing up buffer: path={}, size={}", file.getAbsolutePath(), buffer.remaining()); + public SavedBuffer(File savedFile, List params) { + this.savedFile = savedFile; + this.params = params; + } - FileChannel channel = null; + public void open(Callback callback) { + try { + channel = new RandomAccessFile(savedFile, "rw").getChannel(); + callback.process(params, channel); + success(); + } catch (Exception e) { + LOG.error("Failed to process file. Skipping the file: file={}", savedFile, e); + } finally { try { - channel = new FileOutputStream(file).getChannel(); - channel.write(buffer); - } - catch (Exception e) { - LOG.error("Failed to save buffer to file: params={}, path={}, buffer={}", copiedParams, file.getAbsolutePath(), buffer, e); - } - finally { - if (channel != null) { - try { - channel.close(); - } - catch (IOException e) { - LOG.warn("Failed to close Channel: channel={}", channel); - } - } + close(); + } catch (IOException e) { + LOG.warn("Failed to close file: file={}", savedFile, e); } + } } - public static class SavedBuffer - implements Closeable - { - private final List params; - private final File savedFile; - private FileChannel channel; - - public SavedBuffer(File savedFile, List params) - { - this.savedFile = savedFile; - this.params = params; - } - - public void open(Callback callback) - { - try { - channel = new RandomAccessFile(savedFile, "rw").getChannel(); - callback.process(params, channel); - success(); - } - catch (Exception e) { - LOG.error("Failed to process file. Skipping the file: file={}", savedFile, e); - } - finally { - try { - close(); - } - catch (IOException e) { - LOG.warn("Failed to close file: file={}", savedFile, e); - } - } - } - - public void remove() - { - if (!savedFile.delete()) { - LOG.warn("Failed to delete file: file={}", savedFile); - } - } + public void remove() { + if (!savedFile.delete()) { + LOG.warn("Failed to delete file: file={}", savedFile); + } + } - private void success() - { - try { - close(); - } - catch (IOException e) { - LOG.warn("Failed to close file: file={}", savedFile, e); - } - finally { - remove(); - } - } + private void success() { + try { + close(); + } catch (IOException e) { + LOG.warn("Failed to close file: file={}", savedFile, e); + } finally { + remove(); + } + } - @Override - public void close() - throws IOException - { - if (channel != null && channel.isOpen()) { - channel.close(); - channel = null; - } - } + @Override + public void close() throws IOException { + if (channel != null && channel.isOpen()) { + channel.close(); + channel = null; + } + } - @Override - public String toString() { - return "SavedBuffer{" + - "params=" + params + - ", savedFile=" + savedFile + - '}'; - } + @Override + public String toString() { + return "SavedBuffer{" + "params=" + params + ", savedFile=" + savedFile + '}'; + } - public Path getPath() { - return savedFile.toPath(); - } + public Path getPath() { + return savedFile.toPath(); + } - public interface Callback - { - void process(List params, FileChannel channel); - } + public interface Callback { + void process(List params, FileChannel channel); } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/flusher/Flusher.java b/fluency-core/src/main/java/org/komamitsu/fluency/flusher/Flusher.java index 97a42463..dc6e76c4 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/flusher/Flusher.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/flusher/Flusher.java @@ -16,15 +16,6 @@ package org.komamitsu.fluency.flusher; -import org.komamitsu.fluency.buffer.Buffer; -import org.komamitsu.fluency.ingester.Ingester; -import org.komamitsu.fluency.util.ExecutorServiceUtils; -import org.komamitsu.fluency.validation.Validatable; -import org.komamitsu.fluency.validation.annotation.Max; -import org.komamitsu.fluency.validation.annotation.Min; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.Flushable; import java.util.concurrent.BlockingQueue; @@ -36,271 +27,252 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import org.komamitsu.fluency.buffer.Buffer; +import org.komamitsu.fluency.ingester.Ingester; +import org.komamitsu.fluency.util.ExecutorServiceUtils; +import org.komamitsu.fluency.validation.Validatable; +import org.komamitsu.fluency.validation.annotation.Max; +import org.komamitsu.fluency.validation.annotation.Min; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class Flusher - implements Flushable, Closeable -{ - private static final Logger LOG = LoggerFactory.getLogger(Flusher.class); - private static final int DEFAULT_EVENT_QUEUE_SIZE = 16; - protected final Buffer buffer; - protected final Ingester ingester; - private final AtomicBoolean isTerminated = new AtomicBoolean(); - private final Config config; - private final BlockingQueue eventQueue = new LinkedBlockingQueue<>(DEFAULT_EVENT_QUEUE_SIZE); - private final ExecutorService executorService = ExecutorServiceUtils.newSingleThreadDaemonExecutor(); - - public Flusher(Config config, Buffer buffer, Ingester ingester) - { - config.validateValues(); - this.config = config; - this.buffer = buffer; - this.ingester = ingester; - executorService.execute(this::runLoop); +public class Flusher implements Flushable, Closeable { + private static final Logger LOG = LoggerFactory.getLogger(Flusher.class); + private static final int DEFAULT_EVENT_QUEUE_SIZE = 16; + protected final Buffer buffer; + protected final Ingester ingester; + private final AtomicBoolean isTerminated = new AtomicBoolean(); + private final Config config; + private final BlockingQueue eventQueue = + new LinkedBlockingQueue<>(DEFAULT_EVENT_QUEUE_SIZE); + private final ExecutorService executorService = + ExecutorServiceUtils.newSingleThreadDaemonExecutor(); + + public Flusher(Config config, Buffer buffer, Ingester ingester) { + config.validateValues(); + this.config = config; + this.buffer = buffer; + this.ingester = ingester; + executorService.execute(this::runLoop); + } + + private void runLoop() { + Boolean wakeup = null; + do { + try { + wakeup = eventQueue.poll(config.getFlushAttemptIntervalMillis(), TimeUnit.MILLISECONDS); + boolean force = wakeup != null; + buffer.flush(ingester, force); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Throwable e) { + LOG.error("Failed to flush", e); + } + } while (!executorService.isShutdown()); + + if (wakeup == null) { + // The above run loop can quit without force buffer flush in the following cases + // - close() is called right after the repeated non-force buffer flush executed in the run + // loop + // + // In these cases, remaining buffers won't be flushed. + // So force buffer flush is executed here just in case + try { + buffer.flush(ingester, true); + } catch (Throwable e) { + LOG.error("Failed to flush", e); + } } + } - private void runLoop() { - Boolean wakeup = null; - do { - try { - wakeup = eventQueue.poll(config.getFlushAttemptIntervalMillis(), TimeUnit.MILLISECONDS); - boolean force = wakeup != null; - buffer.flush(ingester, force); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - catch (Throwable e) { - LOG.error("Failed to flush", e); - } - } - while (!executorService.isShutdown()); - - if (wakeup == null) { - // The above run loop can quit without force buffer flush in the following cases - // - close() is called right after the repeated non-force buffer flush executed in the run loop - // - // In these cases, remaining buffers won't be flushed. - // So force buffer flush is executed here just in case - try { - buffer.flush(ingester, true); - } - catch (Throwable e) { - LOG.error("Failed to flush", e); - } - } - } + public Buffer getBuffer() { + return buffer; + } - public Buffer getBuffer() - { - return buffer; - } + @Override + public void flush() { + eventQueue.offer(true); + } - @Override - public void flush() - { - eventQueue.offer(true); - } - - private void flushBufferQuietly() - { - LOG.trace("Flushing the buffer"); + private void flushBufferQuietly() { + LOG.trace("Flushing the buffer"); - try { - flush(); - } - catch (Throwable e) { - LOG.error("Failed to call flush()", e); - } + try { + flush(); + } catch (Throwable e) { + LOG.error("Failed to call flush()", e); } + } - private void finishExecutorQuietly() - { - LOG.trace("Finishing the executor"); + private void finishExecutorQuietly() { + LOG.trace("Finishing the executor"); - ExecutorServiceUtils.finishExecutorService( - executorService, - config.getWaitUntilBufferFlushed()); - } + ExecutorServiceUtils.finishExecutorService(executorService, config.getWaitUntilBufferFlushed()); + } - private void closeBufferQuietly() - { - LOG.trace("Closing the buffer"); + private void closeBufferQuietly() { + LOG.trace("Closing the buffer"); - try { - buffer.close(); - } - catch (Throwable e) { - LOG.warn("Failed to close the buffer", e); - } + try { + buffer.close(); + } catch (Throwable e) { + LOG.warn("Failed to close the buffer", e); } + } - private void closeIngesterQuietly() - { - LOG.trace("Closing the ingester"); - - try { - // Close the socket at the end to prevent the server from failing to read from the connection - ingester.close(); - } - catch (Throwable e) { - LOG.error("Failed to close the ingester", e); - } - } + private void closeIngesterQuietly() { + LOG.trace("Closing the ingester"); - private void closeResourcesQuietly() - { - ExecutorService executorService = Executors.newSingleThreadExecutor(); - try { - Future future = executorService.submit(() -> { + try { + // Close the socket at the end to prevent the server from failing to read from the connection + ingester.close(); + } catch (Throwable e) { + LOG.error("Failed to close the ingester", e); + } + } + + private void closeResourcesQuietly() { + ExecutorService executorService = Executors.newSingleThreadExecutor(); + try { + Future future = + executorService.submit( + () -> { closeBufferQuietly(); closeIngesterQuietly(); isTerminated.set(true); return null; - }); - future.get(getWaitUntilTerminated(), TimeUnit.SECONDS); - } - catch (InterruptedException e) { - LOG.warn("Interrupted", e); - Thread.currentThread().interrupt(); - } - catch (ExecutionException e) { - LOG.warn("closeBuffer() failed", e); - } - catch (TimeoutException e) { - LOG.warn("closeBuffer() timed out", e); - } - finally { - executorService.shutdown();; - } + }); + future.get(getWaitUntilTerminated(), TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOG.warn("Interrupted", e); + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + LOG.warn("closeBuffer() failed", e); + } catch (TimeoutException e) { + LOG.warn("closeBuffer() timed out", e); + } finally { + executorService.shutdown(); + ; } + } + + @Override + public void close() { + flushBufferQuietly(); + + finishExecutorQuietly(); + + closeResourcesQuietly(); + } + + public boolean isTerminated() { + return isTerminated.get(); + } + + public Ingester getIngester() { + return ingester; + } + + /** + * @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} + */ + @Deprecated + public int getFlushIntervalMillis() { + return config.getFlushAttemptIntervalMillis(); + } + + public int getFlushAttemptIntervalMillis() { + return config.getFlushAttemptIntervalMillis(); + } + + public int getWaitUntilBufferFlushed() { + return config.getWaitUntilBufferFlushed(); + } + + public int getWaitUntilTerminated() { + return config.getWaitUntilTerminated(); + } + + @Override + public String toString() { + return "Flusher{" + + "isTerminated=" + + isTerminated + + ", buffer=" + + buffer + + ", ingester=" + + ingester + + ", config=" + + config + + '}'; + } + + public static class Config implements Validatable { + @Min(20) + @Max(2000) + private int flushAttemptIntervalMillis = 600; + + @Min(1) + private int waitUntilBufferFlushed = 60; + + @Min(1) + private int waitUntilTerminated = 60; - @Override - public void close() - { - flushBufferQuietly(); - - finishExecutorQuietly(); + /** + * @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} + */ + @Deprecated + public int getFlushIntervalMillis() { + return flushAttemptIntervalMillis; + } - closeResourcesQuietly(); + /** + * @deprecated As of release 2.4.0, replaced by {@link #setFlushAttemptIntervalMillis(int + * flushAttemptIntervalMillis)} + */ + @Deprecated + public void setFlushIntervalMillis(int flushAttemptIntervalMillis) { + this.flushAttemptIntervalMillis = flushAttemptIntervalMillis; } - public boolean isTerminated() - { - return isTerminated.get(); + public int getFlushAttemptIntervalMillis() { + return flushAttemptIntervalMillis; } - public Ingester getIngester() - { - return ingester; + public void setFlushAttemptIntervalMillis(int flushAttemptIntervalMillis) { + this.flushAttemptIntervalMillis = flushAttemptIntervalMillis; } - /** - * @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} - */ - @Deprecated - public int getFlushIntervalMillis() - { - return config.getFlushAttemptIntervalMillis(); + public int getWaitUntilBufferFlushed() { + return waitUntilBufferFlushed; } - public int getFlushAttemptIntervalMillis() - { - return config.getFlushAttemptIntervalMillis(); + public void setWaitUntilBufferFlushed(int waitUntilBufferFlushed) { + this.waitUntilBufferFlushed = waitUntilBufferFlushed; } - public int getWaitUntilBufferFlushed() - { - return config.getWaitUntilBufferFlushed(); + public int getWaitUntilTerminated() { + return waitUntilTerminated; } - public int getWaitUntilTerminated() - { - return config.getWaitUntilTerminated(); + public void setWaitUntilTerminated(int waitUntilTerminated) { + this.waitUntilTerminated = waitUntilTerminated; } - @Override - public String toString() - { - return "Flusher{" + - "isTerminated=" + isTerminated + - ", buffer=" + buffer + - ", ingester=" + ingester + - ", config=" + config + - '}'; + void validateValues() { + validate(); } - public static class Config - implements Validatable - { - @Min(20) - @Max(2000) - private int flushAttemptIntervalMillis = 600; - @Min(1) - private int waitUntilBufferFlushed = 60; - @Min(1) - private int waitUntilTerminated = 60; - - /** - * @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} - */ - @Deprecated - public int getFlushIntervalMillis() - { - return flushAttemptIntervalMillis; - } - - /** - * @deprecated As of release 2.4.0, replaced by {@link #setFlushAttemptIntervalMillis(int flushAttemptIntervalMillis)} - */ - @Deprecated - public void setFlushIntervalMillis(int flushAttemptIntervalMillis) - { - this.flushAttemptIntervalMillis = flushAttemptIntervalMillis; - } - - public int getFlushAttemptIntervalMillis() - { - return flushAttemptIntervalMillis; - } - - public void setFlushAttemptIntervalMillis(int flushAttemptIntervalMillis) - { - this.flushAttemptIntervalMillis = flushAttemptIntervalMillis; - } - - public int getWaitUntilBufferFlushed() - { - return waitUntilBufferFlushed; - } - - public void setWaitUntilBufferFlushed(int waitUntilBufferFlushed) - { - this.waitUntilBufferFlushed = waitUntilBufferFlushed; - } - - public int getWaitUntilTerminated() - { - return waitUntilTerminated; - } - - public void setWaitUntilTerminated(int waitUntilTerminated) - { - this.waitUntilTerminated = waitUntilTerminated; - } - - void validateValues() - { - validate(); - } - - @Override - public String toString() - { - return "Config{" + - "flushAttemptIntervalMillis=" + flushAttemptIntervalMillis + - ", waitUntilBufferFlushed=" + waitUntilBufferFlushed + - ", waitUntilTerminated=" + waitUntilTerminated + - '}'; - } + @Override + public String toString() { + return "Config{" + + "flushAttemptIntervalMillis=" + + flushAttemptIntervalMillis + + ", waitUntilBufferFlushed=" + + waitUntilBufferFlushed + + ", waitUntilTerminated=" + + waitUntilTerminated + + '}'; } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/ingester/Ingester.java b/fluency-core/src/main/java/org/komamitsu/fluency/ingester/Ingester.java index b84d157f..8c9b09c0 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/ingester/Ingester.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/ingester/Ingester.java @@ -16,17 +16,13 @@ package org.komamitsu.fluency.ingester; -import org.komamitsu.fluency.ingester.sender.Sender; - import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; +import org.komamitsu.fluency.ingester.sender.Sender; -public interface Ingester - extends Closeable -{ - void ingest(String tag, ByteBuffer dataBuffer) - throws IOException; +public interface Ingester extends Closeable { + void ingest(String tag, ByteBuffer dataBuffer) throws IOException; - Sender getSender(); + Sender getSender(); } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/ingester/sender/ErrorHandler.java b/fluency-core/src/main/java/org/komamitsu/fluency/ingester/sender/ErrorHandler.java index 7a6d0d50..3c46f0eb 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/ingester/sender/ErrorHandler.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/ingester/sender/ErrorHandler.java @@ -16,7 +16,6 @@ package org.komamitsu.fluency.ingester.sender; -public interface ErrorHandler -{ - void handle(Throwable e); +public interface ErrorHandler { + void handle(Throwable e); } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/ingester/sender/Sender.java b/fluency-core/src/main/java/org/komamitsu/fluency/ingester/sender/Sender.java index 89314581..3a3d63c1 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/ingester/sender/Sender.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/ingester/sender/Sender.java @@ -16,28 +16,21 @@ package org.komamitsu.fluency.ingester.sender; -public interface Sender -{ - class Config - { - private ErrorHandler errorHandler; +public interface Sender { + class Config { + private ErrorHandler errorHandler; - public ErrorHandler getErrorHandler() - { - return errorHandler; - } + public ErrorHandler getErrorHandler() { + return errorHandler; + } - public void setErrorHandler(ErrorHandler errorHandler) - { - this.errorHandler = errorHandler; - } + public void setErrorHandler(ErrorHandler errorHandler) { + this.errorHandler = errorHandler; + } - @Override - public String toString() - { - return "Config{" + - "senderErrorHandler=" + errorHandler + - '}'; - } + @Override + public String toString() { + return "Config{" + "senderErrorHandler=" + errorHandler + '}'; } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/AbstractRecordFormatter.java b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/AbstractRecordFormatter.java index 0979de04..dff72256 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/AbstractRecordFormatter.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/AbstractRecordFormatter.java @@ -18,55 +18,49 @@ import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; import org.komamitsu.fluency.EventTime; import org.msgpack.jackson.dataformat.MessagePackFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; - -public abstract class AbstractRecordFormatter - implements RecordFormatter -{ - private static final Logger LOG = LoggerFactory.getLogger(AbstractRecordFormatter.class); - protected final ObjectMapper objectMapperForMessagePack = new ObjectMapper(new MessagePackFactory()); - protected final Config config; +public abstract class AbstractRecordFormatter implements RecordFormatter { + private static final Logger LOG = LoggerFactory.getLogger(AbstractRecordFormatter.class); + protected final ObjectMapper objectMapperForMessagePack = + new ObjectMapper(new MessagePackFactory()); + protected final Config config; - public AbstractRecordFormatter(Config config) - { - this.config = config; - registerObjectMapperModules(objectMapperForMessagePack); - } + public AbstractRecordFormatter(Config config) { + this.config = config; + registerObjectMapperModules(objectMapperForMessagePack); + } - public abstract byte[] format(String tag, Object timestamp, Map data); + public abstract byte[] format(String tag, Object timestamp, Map data); - public abstract byte[] formatFromMessagePack(String tag, Object timestamp, byte[] mapValue, int offset, int len); + public abstract byte[] formatFromMessagePack( + String tag, Object timestamp, byte[] mapValue, int offset, int len); - public abstract byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue); + public abstract byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue); - public abstract String formatName(); + public abstract String formatName(); - protected long getEpoch(Object timestamp) - { - if (timestamp instanceof EventTime) { - return ((EventTime) timestamp).getSeconds(); - } - else if (timestamp instanceof Number) { - return ((Number) timestamp).longValue(); - } - else { - LOG.warn("Invalid timestamp. Using current time: timestamp={}", timestamp); - return System.currentTimeMillis() / 1000; - } + protected long getEpoch(Object timestamp) { + if (timestamp instanceof EventTime) { + return ((EventTime) timestamp).getSeconds(); + } else if (timestamp instanceof Number) { + return ((Number) timestamp).longValue(); + } else { + LOG.warn("Invalid timestamp. Using current time: timestamp={}", timestamp); + return System.currentTimeMillis() / 1000; } + } - protected void registerObjectMapperModules(ObjectMapper objectMapper) - { - List jacksonModules = config.getJacksonModules(); - for (Module module : jacksonModules) { - objectMapper.registerModule(module); - } + protected void registerObjectMapperModules(ObjectMapper objectMapper) { + List jacksonModules = config.getJacksonModules(); + for (Module module : jacksonModules) { + objectMapper.registerModule(module); } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/MessagePackRecordFormatter.java b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/MessagePackRecordFormatter.java index 343ad4c6..c5d47326 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/MessagePackRecordFormatter.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/MessagePackRecordFormatter.java @@ -16,85 +16,75 @@ package org.komamitsu.fluency.recordformat; +import java.nio.ByteBuffer; +import java.util.Map; import org.komamitsu.fluency.recordformat.recordaccessor.MapRecordAccessor; import org.komamitsu.fluency.recordformat.recordaccessor.MessagePackRecordAccessor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.ByteBuffer; -import java.util.Map; +public class MessagePackRecordFormatter extends AbstractRecordFormatter { + private static final Logger LOG = LoggerFactory.getLogger(MessagePackRecordFormatter.class); -public class MessagePackRecordFormatter - extends AbstractRecordFormatter -{ - private static final Logger LOG = LoggerFactory.getLogger(MessagePackRecordFormatter.class); + public MessagePackRecordFormatter() { + this(new Config()); + } - public MessagePackRecordFormatter() - { - this(new Config()); - } + public MessagePackRecordFormatter(Config config) { + super(config); + } - public MessagePackRecordFormatter(Config config) - { - super(config); + @Override + public byte[] format(String tag, Object timestamp, Map data) { + try { + MapRecordAccessor recordAccessor = new MapRecordAccessor(data); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return recordAccessor.toMessagePack(objectMapperForMessagePack); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a Map record: cause=%s, tag=%s, timestamp=%s, recordCount=%d", + e.getMessage(), tag, timestamp, data.size())); + throw e; } + } - @Override - public byte[] format(String tag, Object timestamp, Map data) - { - try { - MapRecordAccessor recordAccessor = new MapRecordAccessor(data); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return recordAccessor.toMessagePack(objectMapperForMessagePack); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a Map record: cause=%s, tag=%s, timestamp=%s, recordCount=%d", - e.getMessage(), tag, timestamp, data.size())); - throw e; - } + @Override + public byte[] formatFromMessagePack( + String tag, Object timestamp, byte[] mapValue, int offset, int len) { + try { + MessagePackRecordAccessor recordAccessor = + new MessagePackRecordAccessor(ByteBuffer.wrap(mapValue, offset, len)); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return recordAccessor.toMessagePack(objectMapperForMessagePack); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, offset=%d, len=%d", + e.getMessage(), tag, timestamp, offset, len)); + throw e; } + } - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, byte[] mapValue, int offset, int len) - { - try { - MessagePackRecordAccessor recordAccessor = new MessagePackRecordAccessor(ByteBuffer.wrap(mapValue, offset, len)); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return recordAccessor.toMessagePack(objectMapperForMessagePack); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, offset=%d, len=%d", - e.getMessage(), tag, timestamp, offset, len)); - throw e; - } + @Override + public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) { + try { + MessagePackRecordAccessor recordAccessor = new MessagePackRecordAccessor(mapValue); + recordAccessor.setTimestamp(getEpoch(timestamp)); + return recordAccessor.toMessagePack(objectMapperForMessagePack); + } catch (Throwable e) { + LOG.error( + String.format( + "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, bytebuf=%s", + e.getMessage(), tag, timestamp, mapValue)); + throw e; } + } - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) - { - try { - MessagePackRecordAccessor recordAccessor = new MessagePackRecordAccessor(mapValue); - recordAccessor.setTimestamp(getEpoch(timestamp)); - return recordAccessor.toMessagePack(objectMapperForMessagePack); - } - catch (Throwable e) { - LOG.error(String.format( - "Failed to format a MessagePack record: cause=%s, tag=%s, timestamp=%s, bytebuf=%s", - e.getMessage(), tag, timestamp, mapValue)); - throw e; - } - } - - @Override - public String formatName() - { - return "msgpack"; - } + @Override + public String formatName() { + return "msgpack"; + } - public static class Config - extends RecordFormatter.Config - { - } + public static class Config extends RecordFormatter.Config {} } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/RecordFormatter.java b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/RecordFormatter.java index 996b8c81..393981ff 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/RecordFormatter.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/RecordFormatter.java @@ -17,34 +17,29 @@ package org.komamitsu.fluency.recordformat; import com.fasterxml.jackson.databind.Module; - import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; import java.util.Map; -public interface RecordFormatter -{ - byte[] format(String tag, Object timestamp, Map data); +public interface RecordFormatter { + byte[] format(String tag, Object timestamp, Map data); - byte[] formatFromMessagePack(String tag, Object timestamp, byte[] mapValue, int offset, int len); + byte[] formatFromMessagePack(String tag, Object timestamp, byte[] mapValue, int offset, int len); - byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue); + byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue); - String formatName(); + String formatName(); - class Config - { - private List jacksonModules = Collections.emptyList(); + class Config { + private List jacksonModules = Collections.emptyList(); - public List getJacksonModules() - { - return jacksonModules; - } + public List getJacksonModules() { + return jacksonModules; + } - public void setJacksonModules(List jacksonModules) - { - this.jacksonModules = jacksonModules; - } + public void setJacksonModules(List jacksonModules) { + this.jacksonModules = jacksonModules; } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/MapRecordAccessor.java b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/MapRecordAccessor.java index 26ab9190..0b64b699 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/MapRecordAccessor.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/MapRecordAccessor.java @@ -18,55 +18,45 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; - import java.util.HashMap; import java.util.Map; -public class MapRecordAccessor - implements RecordAccessor -{ - private Map map; - - public MapRecordAccessor(Map map) - { - this.map = new HashMap<>(map); - } +public class MapRecordAccessor implements RecordAccessor { + private Map map; - @Override - public String getAsString(String key) - { - Object value = map.get(key); - if (value == null) { - return null; - } - return value.toString(); - } + public MapRecordAccessor(Map map) { + this.map = new HashMap<>(map); + } - @Override - public void setTimestamp(long timestamp) - { - map.putIfAbsent("time", timestamp); + @Override + public String getAsString(String key) { + Object value = map.get(key); + if (value == null) { + return null; } - - @Override - public byte[] toMessagePack(ObjectMapper objectMapperForMessagePack) - { - try { - return objectMapperForMessagePack.writeValueAsBytes(map); - } - catch (JsonProcessingException e) { - throw new IllegalStateException("Failed to serialize the map to MessagePack format", e); - } + return value.toString(); + } + + @Override + public void setTimestamp(long timestamp) { + map.putIfAbsent("time", timestamp); + } + + @Override + public byte[] toMessagePack(ObjectMapper objectMapperForMessagePack) { + try { + return objectMapperForMessagePack.writeValueAsBytes(map); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Failed to serialize the map to MessagePack format", e); } - - @Override - public String toJson(ObjectMapper objectMapperForJson) - { - try { - return objectMapperForJson.writeValueAsString(map); - } - catch (JsonProcessingException e) { - throw new IllegalStateException("Failed to serialize the map to JSON format", e); - } + } + + @Override + public String toJson(ObjectMapper objectMapperForJson) { + try { + return objectMapperForJson.writeValueAsString(map); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Failed to serialize the map to JSON format", e); } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/MessagePackRecordAccessor.java b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/MessagePackRecordAccessor.java index 7ed29a31..81640474 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/MessagePackRecordAccessor.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/MessagePackRecordAccessor.java @@ -17,6 +17,10 @@ package org.komamitsu.fluency.recordformat.recordaccessor; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; import org.msgpack.core.MessagePack; import org.msgpack.core.MessagePacker; import org.msgpack.core.MessageUnpacker; @@ -24,96 +28,79 @@ import org.msgpack.value.Value; import org.msgpack.value.ValueFactory; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Map; - -public class MessagePackRecordAccessor - implements RecordAccessor -{ - private static final StringValue KEY_TIME = ValueFactory.newString("time"); - private MapValueBytes mapValueBytes; - - private static class MapValueBytes { - private final byte[] byteArray; - private final Map map; +public class MessagePackRecordAccessor implements RecordAccessor { + private static final StringValue KEY_TIME = ValueFactory.newString("time"); + private MapValueBytes mapValueBytes; - MapValueBytes(ByteBuffer byteBuffer) - { - byte[] mapValueBytes = new byte[byteBuffer.remaining()]; - byteBuffer.get(mapValueBytes); - try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(mapValueBytes)) { - this.map = unpacker.unpackValue().asMapValue().map(); - this.byteArray = mapValueBytes; - } - catch (IOException e) { - throw new IllegalArgumentException("Invalid MessagePack ByteBuffer", e); - } - } - - byte[] byteArray() - { - return byteArray; - } + private static class MapValueBytes { + private final byte[] byteArray; + private final Map map; - Map map() - { - return map; - } + MapValueBytes(ByteBuffer byteBuffer) { + byte[] mapValueBytes = new byte[byteBuffer.remaining()]; + byteBuffer.get(mapValueBytes); + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(mapValueBytes)) { + this.map = unpacker.unpackValue().asMapValue().map(); + this.byteArray = mapValueBytes; + } catch (IOException e) { + throw new IllegalArgumentException("Invalid MessagePack ByteBuffer", e); + } } - public MessagePackRecordAccessor(ByteBuffer byteBuffer) - { - mapValueBytes = new MapValueBytes(byteBuffer); + byte[] byteArray() { + return byteArray; } - @Override - public String getAsString(String key) - { - Value value = mapValueBytes.map().get(ValueFactory.newString(key)); - if (value == null) { - return null; - } - return value.toString(); + Map map() { + return map; } + } - @Override - public void setTimestamp(long timestamp) - { - int mapSize = mapValueBytes.map().size(); - try (ByteArrayOutputStream output = new ByteArrayOutputStream(mapValueBytes.byteArray().length + 16)) { - try (MessagePacker packer = MessagePack.newDefaultPacker(output)) { - if (mapValueBytes.map().containsKey(KEY_TIME)) { - packer.packMapHeader(mapSize); - } - else { - packer.packMapHeader(mapSize + 1); - KEY_TIME.writeTo(packer); - packer.packLong(timestamp); - } + public MessagePackRecordAccessor(ByteBuffer byteBuffer) { + mapValueBytes = new MapValueBytes(byteBuffer); + } - for (Map.Entry entry : mapValueBytes.map().entrySet()) { - packer.packValue(entry.getKey()); - packer.packValue(entry.getValue()); - } - } - mapValueBytes = new MapValueBytes(ByteBuffer.wrap(output.toByteArray())); + @Override + public String getAsString(String key) { + Value value = mapValueBytes.map().get(ValueFactory.newString(key)); + if (value == null) { + return null; + } + return value.toString(); + } + + @Override + public void setTimestamp(long timestamp) { + int mapSize = mapValueBytes.map().size(); + try (ByteArrayOutputStream output = + new ByteArrayOutputStream(mapValueBytes.byteArray().length + 16)) { + try (MessagePacker packer = MessagePack.newDefaultPacker(output)) { + if (mapValueBytes.map().containsKey(KEY_TIME)) { + packer.packMapHeader(mapSize); + } else { + packer.packMapHeader(mapSize + 1); + KEY_TIME.writeTo(packer); + packer.packLong(timestamp); } - catch (IOException e) { - throw new IllegalStateException("Failed to upsert `time` field", e); + + for (Map.Entry entry : mapValueBytes.map().entrySet()) { + packer.packValue(entry.getKey()); + packer.packValue(entry.getValue()); } + } + mapValueBytes = new MapValueBytes(ByteBuffer.wrap(output.toByteArray())); + } catch (IOException e) { + throw new IllegalStateException("Failed to upsert `time` field", e); } + } - @Override - public byte[] toMessagePack(ObjectMapper objectMapperForMessagePack) - { - return mapValueBytes.byteArray(); - } + @Override + public byte[] toMessagePack(ObjectMapper objectMapperForMessagePack) { + return mapValueBytes.byteArray(); + } - @Override - public String toJson(ObjectMapper objectMapperForJson) - { - return ValueFactory.newMap(mapValueBytes.map()).toJson(); - } + @Override + public String toJson(ObjectMapper objectMapperForJson) { + return ValueFactory.newMap(mapValueBytes.map()).toJson(); + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/RecordAccessor.java b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/RecordAccessor.java index e79292f7..56502a17 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/RecordAccessor.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/recordformat/recordaccessor/RecordAccessor.java @@ -16,16 +16,14 @@ package org.komamitsu.fluency.recordformat.recordaccessor; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -public interface RecordAccessor -{ - String getAsString(String key); +public interface RecordAccessor { + String getAsString(String key); - void setTimestamp(long timestamp); + void setTimestamp(long timestamp); - byte[] toMessagePack(ObjectMapper objectMapperForMessagePack); + byte[] toMessagePack(ObjectMapper objectMapperForMessagePack); - String toJson(ObjectMapper objectMapperForJson); + String toJson(ObjectMapper objectMapperForJson); } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/util/ExecutorServiceUtils.java b/fluency-core/src/main/java/org/komamitsu/fluency/util/ExecutorServiceUtils.java index 643e84ae..05d9b851 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/util/ExecutorServiceUtils.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/util/ExecutorServiceUtils.java @@ -16,73 +16,65 @@ package org.komamitsu.fluency.util; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class ExecutorServiceUtils -{ - private static final Logger LOG = LoggerFactory.getLogger(ExecutorServiceUtils.class); +public class ExecutorServiceUtils { + private static final Logger LOG = LoggerFactory.getLogger(ExecutorServiceUtils.class); - /** - * Creates an Executor that is based on daemon threads. - * This allows the program to quit without explicitly - * calling shutdown on the pool - * - * @return the newly created single-threaded Executor - */ - public static ExecutorService newSingleThreadDaemonExecutor() { - return Executors.newSingleThreadExecutor(r -> { - Thread t = Executors.defaultThreadFactory().newThread(r); - t.setDaemon(true); - return t; + /** + * Creates an Executor that is based on daemon threads. This allows the program to quit without + * explicitly calling shutdown on the pool + * + * @return the newly created single-threaded Executor + */ + public static ExecutorService newSingleThreadDaemonExecutor() { + return Executors.newSingleThreadExecutor( + r -> { + Thread t = Executors.defaultThreadFactory().newThread(r); + t.setDaemon(true); + return t; }); - } - - /** - * Creates a scheduled thread pool where each thread has the daemon - * property set to true. This allows the program to quit without - * explicitly calling shutdown on the pool - * - * @param corePoolSize the number of threads to keep in the pool, - * even if they are idle + } - * @return a newly created scheduled thread pool - */ - public static ScheduledExecutorService newScheduledDaemonThreadPool(int corePoolSize) { - return Executors.newScheduledThreadPool(corePoolSize, r -> { - Thread t = Executors.defaultThreadFactory().newThread(r); - t.setDaemon(true); - return t; + /** + * Creates a scheduled thread pool where each thread has the daemon property set to true. This + * allows the program to quit without explicitly calling shutdown on the pool + * + * @param corePoolSize the number of threads to keep in the pool, even if they are idle + * @return a newly created scheduled thread pool + */ + public static ScheduledExecutorService newScheduledDaemonThreadPool(int corePoolSize) { + return Executors.newScheduledThreadPool( + corePoolSize, + r -> { + Thread t = Executors.defaultThreadFactory().newThread(r); + t.setDaemon(true); + return t; }); - } + } + public static void finishExecutorService(ExecutorService executorService) { + finishExecutorService(executorService, 3); + } - public static void finishExecutorService(ExecutorService executorService) - { - finishExecutorService(executorService, 3); + public static void finishExecutorService(ExecutorService executorService, long waitSecond) { + executorService.shutdown(); + try { + executorService.awaitTermination(waitSecond, TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOG.warn("1st awaitTermination was interrupted", e); + Thread.currentThread().interrupt(); + } catch (Throwable e) { + LOG.warn("Failed to await the termination of executorService", e); } - public static void finishExecutorService(ExecutorService executorService, long waitSecond) - { - executorService.shutdown(); - try { - executorService.awaitTermination(waitSecond, TimeUnit.SECONDS); - } - catch (InterruptedException e) { - LOG.warn("1st awaitTermination was interrupted", e); - Thread.currentThread().interrupt(); - } - catch (Throwable e) { - LOG.warn("Failed to await the termination of executorService", e); - } - - if (!executorService.isTerminated()) { - executorService.shutdownNow(); - } + if (!executorService.isTerminated()) { + executorService.shutdownNow(); } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/util/Tuple.java b/fluency-core/src/main/java/org/komamitsu/fluency/util/Tuple.java index f8beeab4..80a28846 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/util/Tuple.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/util/Tuple.java @@ -16,24 +16,20 @@ package org.komamitsu.fluency.util; -public class Tuple -{ - private final F first; - private final S second; +public class Tuple { + private final F first; + private final S second; - public Tuple(F first, S second) - { - this.first = first; - this.second = second; - } + public Tuple(F first, S second) { + this.first = first; + this.second = second; + } - public F getFirst() - { - return first; - } + public F getFirst() { + return first; + } - public S getSecond() - { - return second; - } + public S getSecond() { + return second; + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/util/Tuple3.java b/fluency-core/src/main/java/org/komamitsu/fluency/util/Tuple3.java index dfd9543d..47f8ff63 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/util/Tuple3.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/util/Tuple3.java @@ -16,31 +16,26 @@ package org.komamitsu.fluency.util; -public class Tuple3 -{ - private final F first; - private final S second; - private final T third; +public class Tuple3 { + private final F first; + private final S second; + private final T third; - public Tuple3(F first, S second, T third) - { - this.first = first; - this.second = second; - this.third = third; - } + public Tuple3(F first, S second, T third) { + this.first = first; + this.second = second; + this.third = third; + } - public F getFirst() - { - return first; - } + public F getFirst() { + return first; + } - public S getSecond() - { - return second; - } + public S getSecond() { + return second; + } - public T getThird() - { - return third; - } + public T getThird() { + return third; + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/validation/Validatable.java b/fluency-core/src/main/java/org/komamitsu/fluency/validation/Validatable.java index 7a0a326b..5292ce60 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/validation/Validatable.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/validation/Validatable.java @@ -16,137 +16,131 @@ package org.komamitsu.fluency.validation; -import org.komamitsu.fluency.validation.annotation.DecimalMax; -import org.komamitsu.fluency.validation.annotation.DecimalMin; -import org.komamitsu.fluency.validation.annotation.Max; -import org.komamitsu.fluency.validation.annotation.Min; - import java.lang.annotation.Annotation; import java.lang.reflect.Field; import java.math.BigDecimal; import java.util.Arrays; import java.util.List; import java.util.function.BiFunction; +import org.komamitsu.fluency.validation.annotation.DecimalMax; +import org.komamitsu.fluency.validation.annotation.DecimalMin; +import org.komamitsu.fluency.validation.annotation.Max; +import org.komamitsu.fluency.validation.annotation.Min; -public interface Validatable -{ - class ValidationList { - private static class Validation - { - Class annotationClass; - BiFunction isValid; - String messageTemplate; +public interface Validatable { + class ValidationList { + private static class Validation { + Class annotationClass; + BiFunction isValid; + String messageTemplate; - Validation( - Class annotationClass, - BiFunction isValid, - String messageTemplate) - { - this.annotationClass = annotationClass; - this.isValid = isValid; - this.messageTemplate = messageTemplate; - } - } + Validation( + Class annotationClass, + BiFunction isValid, + String messageTemplate) { + this.annotationClass = annotationClass; + this.isValid = isValid; + this.messageTemplate = messageTemplate; + } + } - private static final Validation VALIDATION_MAX = new Validation( - Max.class, - (annotation, actual) -> { - Max maxAnnotation = (Max) annotation; - if (maxAnnotation.inclusive()) { - return maxAnnotation.value() >= actual.longValue(); - } - else { - return maxAnnotation.value() > actual.longValue(); - } - }, - "This field (%s) is more than (%s)"); + private static final Validation VALIDATION_MAX = + new Validation( + Max.class, + (annotation, actual) -> { + Max maxAnnotation = (Max) annotation; + if (maxAnnotation.inclusive()) { + return maxAnnotation.value() >= actual.longValue(); + } else { + return maxAnnotation.value() > actual.longValue(); + } + }, + "This field (%s) is more than (%s)"); - private static final Validation VALIDATION_MIN = new Validation( - Min.class, - (annotation, actual) -> { - Min minAnnotation = (Min) annotation; - if (minAnnotation.inclusive()) { - return minAnnotation.value() <= actual.longValue(); - } - else { - return minAnnotation.value() < actual.longValue(); - } - }, - "This field (%s) is less than (%s)"); + private static final Validation VALIDATION_MIN = + new Validation( + Min.class, + (annotation, actual) -> { + Min minAnnotation = (Min) annotation; + if (minAnnotation.inclusive()) { + return minAnnotation.value() <= actual.longValue(); + } else { + return minAnnotation.value() < actual.longValue(); + } + }, + "This field (%s) is less than (%s)"); - private static final Validation VALIDATION_DECIMAL_MAX = new Validation( - DecimalMax.class, - (annotation, actual) -> { - DecimalMax maxAnnotation = (DecimalMax) annotation; - BigDecimal limitValue = new BigDecimal(maxAnnotation.value()); - BigDecimal actualValue = new BigDecimal(actual.toString()); - if (maxAnnotation.inclusive()) { - return limitValue.compareTo(actualValue) >= 0; - } - else { - return limitValue.compareTo(actualValue) > 0; - } - }, - "This field (%s) is more than (%s)"); + private static final Validation VALIDATION_DECIMAL_MAX = + new Validation( + DecimalMax.class, + (annotation, actual) -> { + DecimalMax maxAnnotation = (DecimalMax) annotation; + BigDecimal limitValue = new BigDecimal(maxAnnotation.value()); + BigDecimal actualValue = new BigDecimal(actual.toString()); + if (maxAnnotation.inclusive()) { + return limitValue.compareTo(actualValue) >= 0; + } else { + return limitValue.compareTo(actualValue) > 0; + } + }, + "This field (%s) is more than (%s)"); - private static final Validation VALIDATION_DECIMAL_MIN = new Validation( - DecimalMin.class, - (annotation, actual) -> { - DecimalMin maxAnnotation = (DecimalMin) annotation; - BigDecimal limitValue = new BigDecimal(maxAnnotation.value()); - BigDecimal actualValue = new BigDecimal(actual.toString()); - if (maxAnnotation.inclusive()) { - return limitValue.compareTo(actualValue) <= 0; - } - else { - return limitValue.compareTo(actualValue) < 0; - } - }, - "This field (%s) is less than (%s)"); + private static final Validation VALIDATION_DECIMAL_MIN = + new Validation( + DecimalMin.class, + (annotation, actual) -> { + DecimalMin maxAnnotation = (DecimalMin) annotation; + BigDecimal limitValue = new BigDecimal(maxAnnotation.value()); + BigDecimal actualValue = new BigDecimal(actual.toString()); + if (maxAnnotation.inclusive()) { + return limitValue.compareTo(actualValue) <= 0; + } else { + return limitValue.compareTo(actualValue) < 0; + } + }, + "This field (%s) is less than (%s)"); - private static final List VALIDATIONS = Arrays.asList( - VALIDATION_MAX, - VALIDATION_MIN, - VALIDATION_DECIMAL_MAX, - VALIDATION_DECIMAL_MIN); - } + private static final List VALIDATIONS = + Arrays.asList( + VALIDATION_MAX, VALIDATION_MIN, VALIDATION_DECIMAL_MAX, VALIDATION_DECIMAL_MIN); + } - default void validate() - { - Class klass = getClass(); - while (klass != Object.class) { - for (Field field : klass.getDeclaredFields()) { - for (ValidationList.Validation validation : ValidationList.VALIDATIONS) { - Class annotationClass = validation.annotationClass; - if (field.isAnnotationPresent(annotationClass)) { - Annotation annotation = field.getAnnotation(annotationClass); - Object value; - try { - field.setAccessible(true); - value = field.get(this); - } - catch (IllegalAccessException e) { - throw new RuntimeException( - String.format("Failed to get a value from field (%s)", field), e); - } + default void validate() { + Class klass = getClass(); + while (klass != Object.class) { + for (Field field : klass.getDeclaredFields()) { + for (ValidationList.Validation validation : ValidationList.VALIDATIONS) { + Class annotationClass = validation.annotationClass; + if (field.isAnnotationPresent(annotationClass)) { + Annotation annotation = field.getAnnotation(annotationClass); + Object value; + try { + field.setAccessible(true); + value = field.get(this); + } catch (IllegalAccessException e) { + throw new RuntimeException( + String.format("Failed to get a value from field (%s)", field), e); + } - if (value == null) { - break; - } + if (value == null) { + break; + } - if (!(value instanceof Number)) { - throw new IllegalArgumentException( - String.format("This field has (%s), but actual field is (%s)", annotation, value.getClass())); - } + if (!(value instanceof Number)) { + throw new IllegalArgumentException( + String.format( + "This field has (%s), but actual field is (%s)", + annotation, value.getClass())); + } - if (!validation.isValid.apply(annotation, (Number) value)) { - throw new IllegalArgumentException( - String.format(validation.messageTemplate, field, value)); - } - } - } + if (!validation.isValid.apply(annotation, (Number) value)) { + throw new IllegalArgumentException( + String.format(validation.messageTemplate, field, value)); } - klass = klass.getSuperclass(); + } } + } + klass = klass.getSuperclass(); } + } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/DecimalMax.java b/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/DecimalMax.java index 932f39bc..134e7a3b 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/DecimalMax.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/DecimalMax.java @@ -25,8 +25,8 @@ @Target(ElementType.FIELD) @Inherited @Retention(RetentionPolicy.RUNTIME) -public @interface DecimalMax -{ - String value(); - boolean inclusive() default true; +public @interface DecimalMax { + String value(); + + boolean inclusive() default true; } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/DecimalMin.java b/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/DecimalMin.java index fb0dfb11..9c1d23d5 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/DecimalMin.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/DecimalMin.java @@ -25,8 +25,8 @@ @Target(ElementType.FIELD) @Inherited @Retention(RetentionPolicy.RUNTIME) -public @interface DecimalMin -{ - String value(); - boolean inclusive() default true; +public @interface DecimalMin { + String value(); + + boolean inclusive() default true; } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/Max.java b/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/Max.java index 828339b2..173b14f3 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/Max.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/Max.java @@ -25,8 +25,8 @@ @Target(ElementType.FIELD) @Inherited @Retention(RetentionPolicy.RUNTIME) -public @interface Max -{ - long value(); - boolean inclusive() default true; +public @interface Max { + long value(); + + boolean inclusive() default true; } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/Min.java b/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/Min.java index dc078327..bdaba7c2 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/Min.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/validation/annotation/Min.java @@ -25,8 +25,8 @@ @Target(ElementType.FIELD) @Inherited @Retention(RetentionPolicy.RUNTIME) -public @interface Min -{ - long value(); - boolean inclusive() default true; +public @interface Min { + long value(); + + boolean inclusive() default true; } diff --git a/fluency-core/src/test/java/org/komamitsu/fluency/FluencyTest.java b/fluency-core/src/test/java/org/komamitsu/fluency/FluencyTest.java index bebfe9d0..65666d9d 100644 --- a/fluency-core/src/test/java/org/komamitsu/fluency/FluencyTest.java +++ b/fluency-core/src/test/java/org/komamitsu/fluency/FluencyTest.java @@ -16,7 +16,21 @@ package org.komamitsu.fluency; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -28,185 +42,151 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; - -class FluencyTest -{ - private static final Logger LOG = LoggerFactory.getLogger(FluencyTest.class); - private Ingester ingester; - private Buffer.Config bufferConfig; - private Flusher.Config flusherConfig; - - @BeforeEach - void setUp() - { - ingester = mock(Ingester.class); - - bufferConfig = new Buffer.Config(); - flusherConfig = new Flusher.Config(); +class FluencyTest { + private static final Logger LOG = LoggerFactory.getLogger(FluencyTest.class); + private Ingester ingester; + private Buffer.Config bufferConfig; + private Flusher.Config flusherConfig; + + @BeforeEach + void setUp() { + ingester = mock(Ingester.class); + + bufferConfig = new Buffer.Config(); + flusherConfig = new Flusher.Config(); + } + + @Test + void testIsTerminated() throws IOException, InterruptedException { + Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); + Flusher flusher = new Flusher(flusherConfig, buffer, ingester); + try (Fluency fluency = new Fluency(buffer, flusher)) { + assertFalse(fluency.isTerminated()); + fluency.close(); + TimeUnit.SECONDS.sleep(1); + assertTrue(fluency.isTerminated()); } - - @Test - void testIsTerminated() - throws IOException, InterruptedException - { - Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); - Flusher flusher = new Flusher(flusherConfig, buffer, ingester); - try (Fluency fluency = new Fluency(buffer, flusher)) { - assertFalse(fluency.isTerminated()); - fluency.close(); - TimeUnit.SECONDS.sleep(1); - assertTrue(fluency.isTerminated()); - } - } - - @Test - void testGetAllocatedBufferSize() - throws IOException - { - Buffer.Config bufferConfig = new Buffer.Config(); - bufferConfig.setChunkInitialSize(1024); - Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); - Flusher flusher = new Flusher(flusherConfig, buffer, ingester); - try (Fluency fluency = new Fluency(buffer, flusher)) { - assertThat(fluency.getAllocatedBufferSize()).isEqualTo(0L); - fluency.emit("foodb.bartbl", ImmutableMap.of("comment", "hello, world")); - assertThat(fluency.getAllocatedBufferSize()).isEqualTo(1024L); - } + } + + @Test + void testGetAllocatedBufferSize() throws IOException { + Buffer.Config bufferConfig = new Buffer.Config(); + bufferConfig.setChunkInitialSize(1024); + Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); + Flusher flusher = new Flusher(flusherConfig, buffer, ingester); + try (Fluency fluency = new Fluency(buffer, flusher)) { + assertThat(fluency.getAllocatedBufferSize()).isEqualTo(0L); + fluency.emit("foodb.bartbl", ImmutableMap.of("comment", "hello, world")); + assertThat(fluency.getAllocatedBufferSize()).isEqualTo(1024L); } - - @ParameterizedTest - @CsvSource({"1, false", "3, true"}) - void testWaitUntilFlusherTerminated(int waitUntilFlusherTerm, boolean expected) - throws IOException, InterruptedException - { - flusherConfig.setWaitUntilTerminated(1); - - // Wait before actually closing in Buffer - int waitBeforeCloseMillis = 2000; - Buffer buffer = spy(new Buffer(bufferConfig, new JsonRecordFormatter())); - doAnswer((invocation) -> { - long start = System.currentTimeMillis(); - try { + } + + @ParameterizedTest + @CsvSource({"1, false", "3, true"}) + void testWaitUntilFlusherTerminated(int waitUntilFlusherTerm, boolean expected) + throws IOException, InterruptedException { + flusherConfig.setWaitUntilTerminated(1); + + // Wait before actually closing in Buffer + int waitBeforeCloseMillis = 2000; + Buffer buffer = spy(new Buffer(bufferConfig, new JsonRecordFormatter())); + doAnswer( + (invocation) -> { + long start = System.currentTimeMillis(); + try { TimeUnit.MILLISECONDS.sleep(waitBeforeCloseMillis); - } - catch (InterruptedException e) { + } catch (InterruptedException e) { long rest = waitBeforeCloseMillis - (System.currentTimeMillis() - start); if (rest > 0) { - try { - TimeUnit.MILLISECONDS.sleep(rest); - } - catch (InterruptedException e1) { - } + try { + TimeUnit.MILLISECONDS.sleep(rest); + } catch (InterruptedException e1) { + } } - } - return null; - }).doCallRealMethod().when(buffer).close(); + } + return null; + }) + .doCallRealMethod() + .when(buffer) + .close(); + + Flusher flusher = new Flusher(flusherConfig, buffer, ingester); + Fluency fluency = new Fluency(buffer, flusher); + + fluency.emit("foo.bar", new HashMap<>()); + fluency.close(); + assertThat(fluency.waitUntilFlusherTerminated(waitUntilFlusherTerm)).isEqualTo(expected); + } + + @ParameterizedTest + @CsvSource({"1, false", "3, true"}) + void testWaitUntilFlushingAllBuffer(int waitUntilFlusherTerm, boolean expected) + throws IOException, InterruptedException { + flusherConfig.setFlushAttemptIntervalMillis(2000); + + Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); + Flusher flusher = new Flusher(flusherConfig, buffer, ingester); + try (Fluency fluency = new Fluency(buffer, flusher)) { + fluency.emit("foo.bar", new HashMap<>()); + assertThat(fluency.waitUntilAllBufferFlushed(waitUntilFlusherTerm)).isEqualTo(expected); + } + } + + @Test + public void testBufferFullException() throws IOException { + final CountDownLatch latch = new CountDownLatch(1); + StuckIngester stuckIngester = new StuckIngester(latch); + + bufferConfig.setChunkInitialSize(64); + bufferConfig.setChunkExpandRatio(2); + bufferConfig.setMaxBufferSize(256); + bufferConfig.setChunkRetentionSize(196); + flusherConfig.setFlushAttemptIntervalMillis(1000); + + Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); + Flusher flusher = new Flusher(flusherConfig, buffer, stuckIngester); + try (Fluency fluency = new Fluency(buffer, flusher)) { + Map event = new HashMap<>(); + event.put("name", "xxxx"); // '{"name":"xxxx"}' (length: 15 bytes) + // Buffers: 64 + 128 = 192 + // 64 + 128 + 256 = 448 > 256 + // 15 * (8 + 1) = 135 + for (int i = 0; i < 8; i++) { + fluency.emit("tag", event); + } + try { + fluency.emit("tag", event); + fail(); + } catch (BufferFullException e) { + assertTrue(true); + } finally { + latch.countDown(); + } + } + } - Flusher flusher = new Flusher(flusherConfig, buffer, ingester); - Fluency fluency = new Fluency(buffer, flusher); + static class StuckIngester implements Ingester { + private final CountDownLatch latch; - fluency.emit("foo.bar", new HashMap<>()); - fluency.close(); - assertThat(fluency.waitUntilFlusherTerminated(waitUntilFlusherTerm)).isEqualTo(expected); + StuckIngester(CountDownLatch latch) { + this.latch = latch; } - @ParameterizedTest - @CsvSource({"1, false", "3, true"}) - void testWaitUntilFlushingAllBuffer(int waitUntilFlusherTerm, boolean expected) - throws IOException, InterruptedException - { - flusherConfig.setFlushAttemptIntervalMillis(2000); - - Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); - Flusher flusher = new Flusher(flusherConfig, buffer, ingester); - try (Fluency fluency = new Fluency(buffer, flusher)) { - fluency.emit("foo.bar", new HashMap<>()); - assertThat(fluency.waitUntilAllBufferFlushed(waitUntilFlusherTerm)).isEqualTo(expected); - } + @Override + public void ingest(String tag, ByteBuffer dataBuffer) { + try { + latch.await(); + } catch (InterruptedException e) { + FluencyTest.LOG.warn("Interrupted in send()", e); + } } - @Test - public void testBufferFullException() - throws IOException - { - final CountDownLatch latch = new CountDownLatch(1); - StuckIngester stuckIngester = new StuckIngester(latch); - - bufferConfig.setChunkInitialSize(64); - bufferConfig.setChunkExpandRatio(2); - bufferConfig.setMaxBufferSize(256); - bufferConfig.setChunkRetentionSize(196); - flusherConfig.setFlushAttemptIntervalMillis(1000); - - Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); - Flusher flusher = new Flusher(flusherConfig, buffer, stuckIngester); - try (Fluency fluency = new Fluency(buffer, flusher)) { - Map event = new HashMap<>(); - event.put("name", "xxxx"); // '{"name":"xxxx"}' (length: 15 bytes) - // Buffers: 64 + 128 = 192 - // 64 + 128 + 256 = 448 > 256 - // 15 * (8 + 1) = 135 - for (int i = 0; i < 8; i++) { - fluency.emit("tag", event); - } - try { - fluency.emit("tag", event); - fail(); - } - catch (BufferFullException e) { - assertTrue(true); - } - finally { - latch.countDown(); - } - } + @Override + public Sender getSender() { + return null; } - static class StuckIngester - implements Ingester - { - private final CountDownLatch latch; - - StuckIngester(CountDownLatch latch) - { - this.latch = latch; - } - - @Override - public void ingest(String tag, ByteBuffer dataBuffer) - { - try { - latch.await(); - } - catch (InterruptedException e) { - FluencyTest.LOG.warn("Interrupted in send()", e); - } - } - - @Override - public Sender getSender() - { - return null; - } - - @Override - public void close() - throws IOException - { - } - } + @Override + public void close() throws IOException {} + } } diff --git a/fluency-core/src/test/java/org/komamitsu/fluency/JsonRecordFormatter.java b/fluency-core/src/test/java/org/komamitsu/fluency/JsonRecordFormatter.java index a99047f9..64e77d17 100644 --- a/fluency-core/src/test/java/org/komamitsu/fluency/JsonRecordFormatter.java +++ b/fluency-core/src/test/java/org/komamitsu/fluency/JsonRecordFormatter.java @@ -18,47 +18,39 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import org.komamitsu.fluency.recordformat.AbstractRecordFormatter; - import java.nio.ByteBuffer; import java.util.Map; +import org.komamitsu.fluency.recordformat.AbstractRecordFormatter; -public class JsonRecordFormatter - extends AbstractRecordFormatter -{ - private ObjectMapper objectMapper = new ObjectMapper(); - - public JsonRecordFormatter() - { - super(new Config()); - } - - @Override - public byte[] format(String tag, Object timestamp, Map data) - { - try { - return objectMapper.writeValueAsBytes(data); - } - catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } +public class JsonRecordFormatter extends AbstractRecordFormatter { + private ObjectMapper objectMapper = new ObjectMapper(); - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, byte[] mapValue, int offset, int len) - { - throw new RuntimeException("Shouldn't be called"); - } - - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) - { - throw new RuntimeException("Shouldn't be called"); - } + public JsonRecordFormatter() { + super(new Config()); + } - @Override - public String formatName() - { - return "json"; + @Override + public byte[] format(String tag, Object timestamp, Map data) { + try { + return objectMapper.writeValueAsBytes(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); } + } + + @Override + public byte[] formatFromMessagePack( + String tag, Object timestamp, byte[] mapValue, int offset, int len) { + throw new RuntimeException("Shouldn't be called"); + } + + @Override + public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) { + throw new RuntimeException("Shouldn't be called"); + } + + @Override + public String formatName() { + return "json"; + } } diff --git a/fluency-core/src/test/java/org/komamitsu/fluency/buffer/BufferPoolTest.java b/fluency-core/src/test/java/org/komamitsu/fluency/buffer/BufferPoolTest.java index 6dbd44b8..40207744 100644 --- a/fluency-core/src/test/java/org/komamitsu/fluency/buffer/BufferPoolTest.java +++ b/fluency-core/src/test/java/org/komamitsu/fluency/buffer/BufferPoolTest.java @@ -16,9 +16,11 @@ package org.komamitsu.fluency.buffer; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.nio.ByteBuffer; @@ -29,154 +31,144 @@ import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class BufferPoolTest -{ - private static final Logger LOG = LoggerFactory.getLogger(BufferPoolTest.class); - - @Test - void testAcquireAndRelease() - throws IOException - { - BufferPool bufferPool = new BufferPool(8 * 1024, 256 * 1024); - ByteBuffer buffer0 = bufferPool.acquireBuffer(100 * 1024); - assertEquals(128 * 1024, buffer0.capacity()); - assertEquals(0, buffer0.position()); - assertEquals(128 * 1024, bufferPool.getAllocatedSize()); - - ByteBuffer buffer1 = bufferPool.acquireBuffer(40 * 1024); - assertEquals(64 * 1024, buffer1.capacity()); - assertEquals(0, buffer1.position()); - assertEquals(192 * 1024, bufferPool.getAllocatedSize()); - - bufferPool.returnBuffer(buffer0); - assertEquals(192 * 1024, bufferPool.getAllocatedSize()); - - ByteBuffer buffer2 = bufferPool.acquireBuffer(20 * 1024); - assertEquals(32 * 1024, buffer2.capacity()); - assertEquals(0, buffer2.position()); - assertEquals(224 * 1024, bufferPool.getAllocatedSize()); - - bufferPool.returnBuffer(buffer2); - assertEquals(224 * 1024, bufferPool.getAllocatedSize()); - - bufferPool.returnBuffer(buffer1); - assertEquals(224 * 1024, bufferPool.getAllocatedSize()); - - long totalBufferSize = getActualTotalBufferSize(bufferPool); - assertEquals(224 * 1024, totalBufferSize); - - bufferPool.releaseBuffers(); - assertEquals(0, bufferPool.getAllocatedSize()); - totalBufferSize = getActualTotalBufferSize(bufferPool); - assertEquals(0, totalBufferSize); - } - - @Test - void testAcquireWithBufferFull() - { - BufferPool bufferPool = new BufferPool(8 * 1024, 256 * 1024); - ByteBuffer buffer = bufferPool.acquireBuffer(64 * 1024); - assertEquals(64 * 1024, buffer.capacity()); - assertEquals(0, buffer.position()); - assertEquals(64 * 1024, bufferPool.getAllocatedSize()); - - buffer = bufferPool.acquireBuffer(64 * 1024); - assertEquals(64 * 1024, buffer.capacity()); - assertEquals(0, buffer.position()); - assertEquals(128 * 1024, bufferPool.getAllocatedSize()); - - buffer = bufferPool.acquireBuffer(64 * 1024); - assertEquals(64 * 1024, buffer.capacity()); - assertEquals(0, buffer.position()); - assertEquals(192 * 1024, bufferPool.getAllocatedSize()); - - buffer = bufferPool.acquireBuffer(64 * 1024); - assertEquals(64 * 1024, buffer.capacity()); - assertEquals(0, buffer.position()); - assertEquals(256 * 1024, bufferPool.getAllocatedSize()); - - buffer = bufferPool.acquireBuffer(64 * 1024); - assertNull(buffer); - } - - @Test - void testAcquireAndReleaseWithMultiThread() - throws InterruptedException - { - final int concurrency = 8; - final BufferPool bufferPool = new BufferPool(1024, 512 * 1024); - final CountDownLatch countDownLatch = new CountDownLatch(concurrency); - Runnable task = () -> { - LinkedList buffers = new LinkedList<>(); - for (int i = 0; i < 1000; i++) { - if ((i / 10) % 2 == 0) { - int size = ((i % 2) + 1) * 1024; - ByteBuffer buffer = bufferPool.acquireBuffer(size); - assertEquals(size, buffer.capacity()); - assertEquals(0, buffer.position()); - buffers.add(buffer); - } - else { - ByteBuffer buffer = buffers.pollFirst(); - assertNotNull(buffer); - bufferPool.returnBuffer(buffer); - } +class BufferPoolTest { + private static final Logger LOG = LoggerFactory.getLogger(BufferPoolTest.class); + + @Test + void testAcquireAndRelease() throws IOException { + BufferPool bufferPool = new BufferPool(8 * 1024, 256 * 1024); + ByteBuffer buffer0 = bufferPool.acquireBuffer(100 * 1024); + assertEquals(128 * 1024, buffer0.capacity()); + assertEquals(0, buffer0.position()); + assertEquals(128 * 1024, bufferPool.getAllocatedSize()); + + ByteBuffer buffer1 = bufferPool.acquireBuffer(40 * 1024); + assertEquals(64 * 1024, buffer1.capacity()); + assertEquals(0, buffer1.position()); + assertEquals(192 * 1024, bufferPool.getAllocatedSize()); + + bufferPool.returnBuffer(buffer0); + assertEquals(192 * 1024, bufferPool.getAllocatedSize()); + + ByteBuffer buffer2 = bufferPool.acquireBuffer(20 * 1024); + assertEquals(32 * 1024, buffer2.capacity()); + assertEquals(0, buffer2.position()); + assertEquals(224 * 1024, bufferPool.getAllocatedSize()); + + bufferPool.returnBuffer(buffer2); + assertEquals(224 * 1024, bufferPool.getAllocatedSize()); + + bufferPool.returnBuffer(buffer1); + assertEquals(224 * 1024, bufferPool.getAllocatedSize()); + + long totalBufferSize = getActualTotalBufferSize(bufferPool); + assertEquals(224 * 1024, totalBufferSize); + + bufferPool.releaseBuffers(); + assertEquals(0, bufferPool.getAllocatedSize()); + totalBufferSize = getActualTotalBufferSize(bufferPool); + assertEquals(0, totalBufferSize); + } + + @Test + void testAcquireWithBufferFull() { + BufferPool bufferPool = new BufferPool(8 * 1024, 256 * 1024); + ByteBuffer buffer = bufferPool.acquireBuffer(64 * 1024); + assertEquals(64 * 1024, buffer.capacity()); + assertEquals(0, buffer.position()); + assertEquals(64 * 1024, bufferPool.getAllocatedSize()); + + buffer = bufferPool.acquireBuffer(64 * 1024); + assertEquals(64 * 1024, buffer.capacity()); + assertEquals(0, buffer.position()); + assertEquals(128 * 1024, bufferPool.getAllocatedSize()); + + buffer = bufferPool.acquireBuffer(64 * 1024); + assertEquals(64 * 1024, buffer.capacity()); + assertEquals(0, buffer.position()); + assertEquals(192 * 1024, bufferPool.getAllocatedSize()); + + buffer = bufferPool.acquireBuffer(64 * 1024); + assertEquals(64 * 1024, buffer.capacity()); + assertEquals(0, buffer.position()); + assertEquals(256 * 1024, bufferPool.getAllocatedSize()); + + buffer = bufferPool.acquireBuffer(64 * 1024); + assertNull(buffer); + } + + @Test + void testAcquireAndReleaseWithMultiThread() throws InterruptedException { + final int concurrency = 8; + final BufferPool bufferPool = new BufferPool(1024, 512 * 1024); + final CountDownLatch countDownLatch = new CountDownLatch(concurrency); + Runnable task = + () -> { + LinkedList buffers = new LinkedList<>(); + for (int i = 0; i < 1000; i++) { + if ((i / 10) % 2 == 0) { + int size = ((i % 2) + 1) * 1024; + ByteBuffer buffer = bufferPool.acquireBuffer(size); + assertEquals(size, buffer.capacity()); + assertEquals(0, buffer.position()); + buffers.add(buffer); + } else { + ByteBuffer buffer = buffers.pollFirst(); + assertNotNull(buffer); + bufferPool.returnBuffer(buffer); } - assertEquals(0, buffers.size()); - countDownLatch.countDown(); + } + assertEquals(0, buffers.size()); + countDownLatch.countDown(); }; - ExecutorService executorService = Executors.newCachedThreadPool(); - for (int i = 0; i < concurrency; i++) { - executorService.execute(task); - } - assertTrue(countDownLatch.await(10, TimeUnit.SECONDS)); - - long totalBufferSize = getActualTotalBufferSize(bufferPool); - LOG.debug("bufferPool.getAllocatedSize() = {}, actual buffered total size={}", bufferPool.getAllocatedSize(), totalBufferSize); - assertEquals(bufferPool.getAllocatedSize(), totalBufferSize); + ExecutorService executorService = Executors.newCachedThreadPool(); + for (int i = 0; i < concurrency; i++) { + executorService.execute(task); } - - private long getActualTotalBufferSize(BufferPool bufferPool) - { - long totalBufferedSize = 0; - for (Map.Entry> entry : bufferPool.bufferPool.entrySet()) { - for (ByteBuffer buffer : entry.getValue()) { - totalBufferedSize += buffer.capacity(); - } - } - return totalBufferedSize; - } - - @Test - void useOffHeap() - throws IOException - { - BufferPool bufferPool = new BufferPool(8 * 1024, 256 * 1024); - assertFalse(bufferPool.getJvmHeapBufferMode()); - ByteBuffer buffer0 = bufferPool.acquireBuffer(100 * 1024); - assertTrue(buffer0.isDirect()); - assertEquals(128 * 1024, buffer0.capacity()); - assertEquals(0, buffer0.position()); - assertEquals(128 * 1024, bufferPool.getAllocatedSize()); - } - - @Test - void useOnHeap() - throws IOException - { - BufferPool bufferPool = new BufferPool(8 * 1024, 256 * 1024, true); - assertTrue(bufferPool.getJvmHeapBufferMode()); - ByteBuffer buffer0 = bufferPool.acquireBuffer(100 * 1024); - assertFalse(buffer0.isDirect()); - assertEquals(128 * 1024, buffer0.capacity()); - assertEquals(0, buffer0.position()); - assertEquals(128 * 1024, bufferPool.getAllocatedSize()); + assertTrue(countDownLatch.await(10, TimeUnit.SECONDS)); + + long totalBufferSize = getActualTotalBufferSize(bufferPool); + LOG.debug( + "bufferPool.getAllocatedSize() = {}, actual buffered total size={}", + bufferPool.getAllocatedSize(), + totalBufferSize); + assertEquals(bufferPool.getAllocatedSize(), totalBufferSize); + } + + private long getActualTotalBufferSize(BufferPool bufferPool) { + long totalBufferedSize = 0; + for (Map.Entry> entry : + bufferPool.bufferPool.entrySet()) { + for (ByteBuffer buffer : entry.getValue()) { + totalBufferedSize += buffer.capacity(); + } } + return totalBufferedSize; + } + + @Test + void useOffHeap() throws IOException { + BufferPool bufferPool = new BufferPool(8 * 1024, 256 * 1024); + assertFalse(bufferPool.getJvmHeapBufferMode()); + ByteBuffer buffer0 = bufferPool.acquireBuffer(100 * 1024); + assertTrue(buffer0.isDirect()); + assertEquals(128 * 1024, buffer0.capacity()); + assertEquals(0, buffer0.position()); + assertEquals(128 * 1024, bufferPool.getAllocatedSize()); + } + + @Test + void useOnHeap() throws IOException { + BufferPool bufferPool = new BufferPool(8 * 1024, 256 * 1024, true); + assertTrue(bufferPool.getJvmHeapBufferMode()); + ByteBuffer buffer0 = bufferPool.acquireBuffer(100 * 1024); + assertFalse(buffer0.isDirect()); + assertEquals(128 * 1024, buffer0.capacity()); + assertEquals(0, buffer0.position()); + assertEquals(128 * 1024, bufferPool.getAllocatedSize()); + } } diff --git a/fluency-core/src/test/java/org/komamitsu/fluency/buffer/BufferTest.java b/fluency-core/src/test/java/org/komamitsu/fluency/buffer/BufferTest.java index ec02867c..72bab9fb 100644 --- a/fluency-core/src/test/java/org/komamitsu/fluency/buffer/BufferTest.java +++ b/fluency-core/src/test/java/org/komamitsu/fluency/buffer/BufferTest.java @@ -16,10 +16,26 @@ package org.komamitsu.fluency.buffer; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.komamitsu.fluency.JsonRecordFormatter; @@ -31,400 +47,374 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - -class BufferTest -{ - private static final Logger LOG = LoggerFactory.getLogger(BufferTest.class); - private Ingester ingester; - private RecordFormatter recordFormatter; - private Buffer.Config bufferConfig; - - @BeforeEach - void setUp() - { - recordFormatter = new JsonRecordFormatter(); - ingester = mock(Ingester.class); - bufferConfig = new Buffer.Config(); +class BufferTest { + private static final Logger LOG = LoggerFactory.getLogger(BufferTest.class); + private Ingester ingester; + private RecordFormatter recordFormatter; + private Buffer.Config bufferConfig; + + @BeforeEach + void setUp() { + recordFormatter = new JsonRecordFormatter(); + ingester = mock(Ingester.class); + bufferConfig = new Buffer.Config(); + } + + @Test + void testBuffer() throws IOException { + int recordSize = 20; // '{"name":"komamitsu"}' + int tags = 10; + float allocRatio = 2f; + int maxAllocSize = (recordSize * 4) * tags; + + bufferConfig.setChunkInitialSize(recordSize); + bufferConfig.setChunkExpandRatio(allocRatio); + bufferConfig.setMaxBufferSize(maxAllocSize); + bufferConfig.setChunkRetentionSize(maxAllocSize / 2); + Buffer buffer = new Buffer(bufferConfig, recordFormatter); + + assertEquals(0, buffer.getAllocatedSize()); + assertEquals(0, buffer.getBufferedDataSize()); + assertEquals(0f, buffer.getBufferUsage(), 0.001); + + Map data = new HashMap<>(); + data.put("name", "komamitsu"); + for (int i = 0; i < tags; i++) { + buffer.append("foodb.bartbl" + i, 1420070400, data); } - - @Test - void testBuffer() - throws IOException - { - int recordSize = 20; // '{"name":"komamitsu"}' - int tags = 10; - float allocRatio = 2f; - int maxAllocSize = (recordSize * 4) * tags; - - bufferConfig.setChunkInitialSize(recordSize); - bufferConfig.setChunkExpandRatio(allocRatio); - bufferConfig.setMaxBufferSize(maxAllocSize); - bufferConfig.setChunkRetentionSize(maxAllocSize / 2); - Buffer buffer = new Buffer(bufferConfig, recordFormatter); - - assertEquals(0, buffer.getAllocatedSize()); - assertEquals(0, buffer.getBufferedDataSize()); - assertEquals(0f, buffer.getBufferUsage(), 0.001); - - Map data = new HashMap<>(); - data.put("name", "komamitsu"); - for (int i = 0; i < tags; i++) { - buffer.append("foodb.bartbl" + i, 1420070400, data); - } - assertEquals(recordSize * tags, buffer.getAllocatedSize()); - assertEquals(recordSize * tags, buffer.getBufferedDataSize()); - assertEquals(((float) recordSize) * tags / maxAllocSize, buffer.getBufferUsage(), 0.001); - for (int i = 0; i < tags; i++) { - buffer.append("foodb.bartbl" + i, 1420070400, data); - } - float totalAllocatedSize = recordSize * (1 + allocRatio) * tags; - assertEquals(totalAllocatedSize, buffer.getAllocatedSize(), 0.001); - assertEquals(2L * recordSize * tags, buffer.getBufferedDataSize()); - assertEquals(totalAllocatedSize / maxAllocSize, buffer.getBufferUsage(), 0.001); - - buffer.flush(ingester, true); - assertEquals(totalAllocatedSize, buffer.getAllocatedSize(), 0.001); - assertEquals(0, buffer.getBufferedDataSize()); - assertEquals(totalAllocatedSize / maxAllocSize, buffer.getBufferUsage(), 0.001); - - buffer.close(); - assertEquals(0, buffer.getAllocatedSize()); - assertEquals(0, buffer.getBufferedDataSize()); - assertEquals(0, buffer.getBufferUsage(), 0.001); + assertEquals(recordSize * tags, buffer.getAllocatedSize()); + assertEquals(recordSize * tags, buffer.getBufferedDataSize()); + assertEquals(((float) recordSize) * tags / maxAllocSize, buffer.getBufferUsage(), 0.001); + for (int i = 0; i < tags; i++) { + buffer.append("foodb.bartbl" + i, 1420070400, data); + } + float totalAllocatedSize = recordSize * (1 + allocRatio) * tags; + assertEquals(totalAllocatedSize, buffer.getAllocatedSize(), 0.001); + assertEquals(2L * recordSize * tags, buffer.getBufferedDataSize()); + assertEquals(totalAllocatedSize / maxAllocSize, buffer.getBufferUsage(), 0.001); + + buffer.flush(ingester, true); + assertEquals(totalAllocatedSize, buffer.getAllocatedSize(), 0.001); + assertEquals(0, buffer.getBufferedDataSize()); + assertEquals(totalAllocatedSize / maxAllocSize, buffer.getBufferUsage(), 0.001); + + buffer.close(); + assertEquals(0, buffer.getAllocatedSize()); + assertEquals(0, buffer.getBufferedDataSize()); + assertEquals(0, buffer.getBufferUsage(), 0.001); + } + + @Test + void testStrictBufferRetentionSizeManagement() throws IOException { + int recordSize = 20; // '{"name":"komamitsu"}' + int initSize = 15; + float allocRatio = 2f; + // 15 -> 30 -> 60 -> 120 -> 240 + int maxAllocSize = (int) (initSize * allocRatio * allocRatio * allocRatio * allocRatio); + String tag = "foodb.bartbl"; + + int retentionSize = 60; + + bufferConfig.setChunkInitialSize(initSize); + bufferConfig.setChunkExpandRatio(allocRatio); + bufferConfig.setMaxBufferSize(maxAllocSize); + bufferConfig.setChunkRetentionSize(retentionSize); + Buffer buffer = new Buffer(bufferConfig, recordFormatter); + + Map data = new HashMap<>(); + data.put("name", "komamitsu"); + + // Buffered data size is 0 and additional 20 bytes data will be buffered (total size: 20 bytes). + buffer.append(tag, 1420070400, data); + buffer.flush(ingester, false); + verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); + assertEquals(20, buffer.getBufferedDataSize()); + + // Buffered data size is 20 and additional 20 bytes data will be buffered (total size: 40 + // bytes). + buffer.append(tag, 1420070400, data); + buffer.flush(ingester, false); + verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); + assertEquals(40, buffer.getBufferedDataSize()); + + // Buffered data size is 40 and additional 20 bytes data will be buffered (total size: 60 + // bytes). + buffer.append(tag, 1420070400, data); + buffer.flush(ingester, false); + verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); + assertEquals(60, buffer.getBufferedDataSize()); + + // Buffered data size is 60 and additional 20 bytes exceeds the retention size. + // The buffered 60 bytes data will be flushed and the additional data will be buffered (total + // size: 20 bytes). + doAnswer( + invocation -> { + String receivedTag = invocation.getArgument(0); + ByteBuffer receivedBuffer = invocation.getArgument(1); + assertEquals(tag, receivedTag); + assertEquals(60, receivedBuffer.remaining()); + return null; + }) + .when(ingester) + .ingest(anyString(), any(ByteBuffer.class)); + + buffer.append(tag, 1420070400, data); + buffer.flush(ingester, false); + verify(ingester, times(1)).ingest(anyString(), any(ByteBuffer.class)); + assertEquals(20, buffer.getBufferedDataSize()); + } + + @Test + void flush() throws IOException { + Buffer buffer = new Buffer(bufferConfig, recordFormatter); + + Ingester ingester = mock(Ingester.class); + buffer.flush(ingester, false); + + verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); + + Map data = new HashMap<>(); + data.put("name", "komamitsu"); + buffer.append("foodb.bartbl", 1420070400, data); + + buffer.flush(ingester, false); + + verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); + + buffer.flush(ingester, true); + + verify(ingester, times(1)).ingest(eq("foodb.bartbl"), any(ByteBuffer.class)); + } + + @Test + void flushWithInterrupt() throws IOException { + Buffer buffer = new Buffer(bufferConfig, recordFormatter); + + Ingester ingester = mock(Ingester.class); + AtomicReference receivedBuffer = new AtomicReference<>(); + doAnswer( + (Answer) + invocation -> { + receivedBuffer.set(((ByteBuffer) invocation.getArgument(1)).duplicate()); + return null; + }) + .when(ingester) + .ingest(anyString(), any(ByteBuffer.class)); + + Map data = new HashMap<>(); + data.put("name", "komamitsu"); + buffer.append("foodb.bartbl", 1420070400, data); + + try { + Thread.currentThread().interrupt(); + buffer.flush(ingester, true); + fail(); + } catch (IOException e) { + // Caused by interruption. This is expected. } - @Test - void testStrictBufferRetentionSizeManagement() - throws IOException - { - int recordSize = 20; // '{"name":"komamitsu"}' - int initSize = 15; - float allocRatio = 2f; - // 15 -> 30 -> 60 -> 120 -> 240 - int maxAllocSize = (int) (initSize * allocRatio * allocRatio * allocRatio * allocRatio); - String tag = "foodb.bartbl"; - - int retentionSize = 60; - - bufferConfig.setChunkInitialSize(initSize); - bufferConfig.setChunkExpandRatio(allocRatio); - bufferConfig.setMaxBufferSize(maxAllocSize); - bufferConfig.setChunkRetentionSize(retentionSize); - Buffer buffer = new Buffer(bufferConfig, recordFormatter); - - Map data = new HashMap<>(); - data.put("name", "komamitsu"); - - // Buffered data size is 0 and additional 20 bytes data will be buffered (total size: 20 bytes). - buffer.append(tag, 1420070400, data); - buffer.flush(ingester, false); - verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); - assertEquals(20, buffer.getBufferedDataSize()); - - // Buffered data size is 20 and additional 20 bytes data will be buffered (total size: 40 bytes). - buffer.append(tag, 1420070400, data); - buffer.flush(ingester, false); - verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); - assertEquals(40, buffer.getBufferedDataSize()); - - // Buffered data size is 40 and additional 20 bytes data will be buffered (total size: 60 bytes). - buffer.append(tag, 1420070400, data); - buffer.flush(ingester, false); - verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); - assertEquals(60, buffer.getBufferedDataSize()); - - // Buffered data size is 60 and additional 20 bytes exceeds the retention size. - // The buffered 60 bytes data will be flushed and the additional data will be buffered (total size: 20 bytes). - doAnswer(invocation -> { - String receivedTag = invocation.getArgument(0); - ByteBuffer receivedBuffer = invocation.getArgument(1); - assertEquals(tag, receivedTag); - assertEquals(60, receivedBuffer.remaining()); - return null; - }).when(ingester).ingest(anyString(), any(ByteBuffer.class)); - - buffer.append(tag, 1420070400, data); - buffer.flush(ingester, false); - verify(ingester, times(1)).ingest(anyString(), any(ByteBuffer.class)); - assertEquals(20, buffer.getBufferedDataSize()); + // Retry + buffer.flush(ingester, true); + + verify(ingester, times(1)).ingest(eq("foodb.bartbl"), any(ByteBuffer.class)); + ObjectMapper objectMapper = new ObjectMapper(); + byte[] receivedData = new byte[receivedBuffer.get().remaining()]; + receivedBuffer.get().get(receivedData); + Map deserialized = + objectMapper.readValue(receivedData, new TypeReference>() {}); + System.out.println(deserialized); + } + + @Test + void testFileBackup() throws IOException { + bufferConfig.setFileBackupDir(System.getProperty("java.io.tmpdir")); + bufferConfig.setFileBackupPrefix("FileBackupTest"); + + // Just for cleaning backup files + try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { + buffer.clearBackupFiles(); + } + try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { + assertEquals(0, buffer.getBufferedDataSize()); } - @Test - void flush() - throws IOException - { - Buffer buffer = new Buffer(bufferConfig, recordFormatter); + long currentTime = System.currentTimeMillis() / 1000; + Map event0 = ImmutableMap.of("name", "a", "age", 42); + Map event1 = ImmutableMap.of("name", "b", "age", 99); + try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { + buffer.append("foo", currentTime, event0); + buffer.append("bar", currentTime, event1); + } - Ingester ingester = mock(Ingester.class); - buffer.flush(ingester, false); + Ingester ingester = mock(Ingester.class); + try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { + buffer.flushInternal(ingester, true); + } - verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); + ObjectMapper objectMapper = new ObjectMapper(); + for (Tuple> tagAndEvent : + ImmutableList.of(new Tuple<>("foo", event0), new Tuple<>("bar", event1))) { + ArgumentCaptor byteBufferArgumentCaptor = + ArgumentCaptor.forClass(ByteBuffer.class); + verify(ingester, times(1)) + .ingest(eq(tagAndEvent.getFirst()), byteBufferArgumentCaptor.capture()); + ByteBuffer byteBuffer = byteBufferArgumentCaptor.getValue(); + byte[] bytes = new byte[byteBuffer.remaining()]; + byteBuffer.get(bytes); + Map map = + objectMapper.readValue(bytes, new TypeReference>() {}); + assertEquals(tagAndEvent.getSecond(), map); + } + } - Map data = new HashMap<>(); - data.put("name", "komamitsu"); - buffer.append("foodb.bartbl", 1420070400, data); + @Test + void testFileBackupThatIsNotDirectory() throws IOException { + File backupDirFile = File.createTempFile("testFileBackupWithInvalidDir", ".tmp"); + backupDirFile.deleteOnExit(); - buffer.flush(ingester, false); + bufferConfig.setFileBackupDir(backupDirFile.getAbsolutePath()); - verify(ingester, times(0)).ingest(anyString(), any(ByteBuffer.class)); + try (Buffer ignored = new Buffer(bufferConfig, recordFormatter)) { + fail(); + } catch (IllegalArgumentException e) { + assertTrue(true); + } + } + + @Test + void testFileBackupThatIsWritable() { + File backupDir = + new File( + System.getProperties().getProperty("java.io.tmpdir"), UUID.randomUUID().toString()); + if (backupDir.mkdir()) { + LOG.info("Created directory: {}", backupDir); + } + if (!backupDir.setWritable(false)) { + throw new RuntimeException("Failed to revoke writable permission"); + } + backupDir.deleteOnExit(); - buffer.flush(ingester, true); + bufferConfig.setFileBackupDir(backupDir.getAbsolutePath()); - verify(ingester, times(1)).ingest(eq("foodb.bartbl"), any(ByteBuffer.class)); + try (Buffer ignored = new Buffer(bufferConfig, recordFormatter)) { + fail(); + } catch (IllegalArgumentException e) { + assertTrue(true); } - - @Test - void flushWithInterrupt() - throws IOException - { - Buffer buffer = new Buffer(bufferConfig, recordFormatter); - - Ingester ingester = mock(Ingester.class); - AtomicReference receivedBuffer = new AtomicReference<>(); - doAnswer((Answer) invocation -> { - receivedBuffer.set(((ByteBuffer)invocation.getArgument(1)).duplicate()); - return null; - }).when(ingester).ingest(anyString(), any(ByteBuffer.class)); - - Map data = new HashMap<>(); - data.put("name", "komamitsu"); - buffer.append("foodb.bartbl", 1420070400, data); - - try { - Thread.currentThread().interrupt(); - buffer.flush(ingester, true); - fail(); - } - catch (IOException e) { - // Caused by interruption. This is expected. - } - - // Retry - buffer.flush(ingester, true); - - verify(ingester, times(1)).ingest(eq("foodb.bartbl"), any(ByteBuffer.class)); - ObjectMapper objectMapper = new ObjectMapper(); - byte[] receivedData = new byte[receivedBuffer.get().remaining()]; - receivedBuffer.get().get(receivedData); - Map deserialized = objectMapper.readValue(receivedData, new TypeReference>() {}); - System.out.println(deserialized); + } + + @Test + void testFileBackupThatIsReadable() { + File backupDir = + new File( + System.getProperties().getProperty("java.io.tmpdir"), UUID.randomUUID().toString()); + if (backupDir.mkdir()) { + LOG.info("Created directory: {}", backupDir); } - - @Test - void testFileBackup() - throws IOException - { - bufferConfig.setFileBackupDir(System.getProperty("java.io.tmpdir")); - bufferConfig.setFileBackupPrefix("FileBackupTest"); - - // Just for cleaning backup files - try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { - buffer.clearBackupFiles(); - } - try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { - assertEquals(0, buffer.getBufferedDataSize()); - } - - long currentTime = System.currentTimeMillis() / 1000; - Map event0 = ImmutableMap.of("name", "a", "age", 42); - Map event1 = ImmutableMap.of("name", "b", "age", 99); - try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { - buffer.append("foo", currentTime, event0); - buffer.append("bar", currentTime, event1); - } - - Ingester ingester = mock(Ingester.class); - try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { - buffer.flushInternal(ingester, true); - } - - ObjectMapper objectMapper = new ObjectMapper(); - for (Tuple> tagAndEvent : - ImmutableList.of(new Tuple<>("foo", event0), new Tuple<>("bar", event1))) { - ArgumentCaptor byteBufferArgumentCaptor = ArgumentCaptor.forClass(ByteBuffer.class); - verify(ingester, times(1)).ingest(eq(tagAndEvent.getFirst()), byteBufferArgumentCaptor.capture()); - ByteBuffer byteBuffer = byteBufferArgumentCaptor.getValue(); - byte[] bytes = new byte[byteBuffer.remaining()]; - byteBuffer.get(bytes); - Map map = objectMapper.readValue(bytes, new TypeReference>() {}); - assertEquals(tagAndEvent.getSecond(), map); - } + if (!backupDir.setReadable(false)) { + throw new RuntimeException("Failed to revoke readable permission"); } + backupDir.deleteOnExit(); - @Test - void testFileBackupThatIsNotDirectory() - throws IOException - { - File backupDirFile = File.createTempFile("testFileBackupWithInvalidDir", ".tmp"); - backupDirFile.deleteOnExit(); - - bufferConfig.setFileBackupDir(backupDirFile.getAbsolutePath()); + bufferConfig.setFileBackupDir(backupDir.getAbsolutePath()); - try (Buffer ignored = new Buffer(bufferConfig, recordFormatter)) { - fail(); - } - catch (IllegalArgumentException e) { - assertTrue(true); - } + try (Buffer ignored = new Buffer(bufferConfig, recordFormatter)) { + fail(); + } catch (IllegalArgumentException e) { + assertTrue(true); } - - @Test - void testFileBackupThatIsWritable() - { - File backupDir = new File(System.getProperties().getProperty("java.io.tmpdir"), UUID.randomUUID().toString()); - if (backupDir.mkdir()) { - LOG.info("Created directory: {}", backupDir); - } - if (!backupDir.setWritable(false)) { - throw new RuntimeException("Failed to revoke writable permission"); - } - backupDir.deleteOnExit(); - - bufferConfig.setFileBackupDir(backupDir.getAbsolutePath()); - - try (Buffer ignored = new Buffer(bufferConfig, recordFormatter)) { - fail(); - } - catch (IllegalArgumentException e) { - assertTrue(true); - } + } + + @Test + void testGetAllocatedSize() throws IOException { + bufferConfig.setChunkInitialSize(256 * 1024); + try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { + assertThat(buffer.getAllocatedSize()).isEqualTo(0L); + Map map = new HashMap<>(); + map.put("name", "komamitsu"); + for (int i = 0; i < 10; i++) { + buffer.append("foo.bar", new Date().getTime(), map); + } + assertThat(buffer.getAllocatedSize()).isEqualTo(256 * 1024L); } - - @Test - void testFileBackupThatIsReadable() - { - File backupDir = new File(System.getProperties().getProperty("java.io.tmpdir"), UUID.randomUUID().toString()); - if (backupDir.mkdir()) { - LOG.info("Created directory: {}", backupDir); - } - if (!backupDir.setReadable(false)) { - throw new RuntimeException("Failed to revoke readable permission"); - } - backupDir.deleteOnExit(); - - bufferConfig.setFileBackupDir(backupDir.getAbsolutePath()); - - try (Buffer ignored = new Buffer(bufferConfig, recordFormatter)) { - fail(); - } - catch (IllegalArgumentException e) { - assertTrue(true); - } + } + + @Test + void testGetBufferedDataSize() throws IOException { + bufferConfig.setChunkInitialSize(256 * 1024); + try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { + assertThat(buffer.getBufferedDataSize()).isEqualTo(0L); + + Map map = new HashMap<>(); + map.put("name", "komamitsu"); + for (int i = 0; i < 10; i++) { + buffer.append("foo.bar", new Date().getTime(), map); + } + assertThat(buffer.getBufferedDataSize()).isGreaterThan(0L); + assertThat(buffer.getBufferedDataSize()).isLessThan(512L); + + buffer.flush(ingester, true); + assertThat(buffer.getBufferedDataSize()).isEqualTo(0L); + } + } + + @Test + void testAppendIfItDoesNotThrowBufferOverflow() throws IOException { + bufferConfig.setChunkInitialSize(64 * 1024); + try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { + + StringBuilder buf = new StringBuilder(); + + for (int i = 0; i < 1024 * 60; i++) { + buf.append('x'); + } + String str60kb = buf.toString(); + + for (int i = 0; i < 1024 * 40; i++) { + buf.append('x'); + } + String str100kb = buf.toString(); + + { + Map map = new HashMap<>(); + map.put("k", str60kb); + buffer.append("tag0", new Date().getTime(), map); + } + + { + Map map = new HashMap<>(); + map.put("k", str100kb); + buffer.append("tag0", new Date().getTime(), map); + } } + } - @Test - void testGetAllocatedSize() - throws IOException + @Test + void validateConfig() { { - bufferConfig.setChunkInitialSize(256 * 1024); - try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { - assertThat(buffer.getAllocatedSize()).isEqualTo(0L); - Map map = new HashMap<>(); - map.put("name", "komamitsu"); - for (int i = 0; i < 10; i++) { - buffer.append("foo.bar", new Date().getTime(), map); - } - assertThat(buffer.getAllocatedSize()).isEqualTo(256 * 1024L); - } + Buffer.Config config = new Buffer.Config(); + config.setChunkInitialSize(4 * 1024 * 1024); + config.setChunkRetentionSize(4 * 1024 * 1024); + assertThrows(IllegalArgumentException.class, () -> new Buffer(config, recordFormatter)); } - @Test - void testGetBufferedDataSize() - throws IOException { - bufferConfig.setChunkInitialSize(256 * 1024); - try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { - assertThat(buffer.getBufferedDataSize()).isEqualTo(0L); - - Map map = new HashMap<>(); - map.put("name", "komamitsu"); - for (int i = 0; i < 10; i++) { - buffer.append("foo.bar", new Date().getTime(), map); - } - assertThat(buffer.getBufferedDataSize()).isGreaterThan(0L); - assertThat(buffer.getBufferedDataSize()).isLessThan(512L); - - buffer.flush(ingester, true); - assertThat(buffer.getBufferedDataSize()).isEqualTo(0L); - } + Buffer.Config config = new Buffer.Config(); + config.setChunkRetentionSize(64 * 1024 * 1024); + config.setMaxBufferSize(64 * 1024 * 1024); + assertThrows(IllegalArgumentException.class, () -> new Buffer(config, recordFormatter)); } - @Test - void testAppendIfItDoesNotThrowBufferOverflow() - throws IOException { - bufferConfig.setChunkInitialSize(64 * 1024); - try (Buffer buffer = new Buffer(bufferConfig, recordFormatter)) { - - StringBuilder buf = new StringBuilder(); - - for (int i = 0; i < 1024 * 60; i++) { - buf.append('x'); - } - String str60kb = buf.toString(); - - for (int i = 0; i < 1024 * 40; i++) { - buf.append('x'); - } - String str100kb = buf.toString(); - - { - Map map = new HashMap<>(); - map.put("k", str60kb); - buffer.append("tag0", new Date().getTime(), map); - } - - { - Map map = new HashMap<>(); - map.put("k", str100kb); - buffer.append("tag0", new Date().getTime(), map); - } - } + Buffer.Config config = new Buffer.Config(); + config.setChunkExpandRatio(1.19f); + assertThrows(IllegalArgumentException.class, () -> new Buffer(config, recordFormatter)); } - @Test - void validateConfig() { - { - Buffer.Config config = new Buffer.Config(); - config.setChunkInitialSize(4 * 1024 * 1024); - config.setChunkRetentionSize(4 * 1024 * 1024); - assertThrows(IllegalArgumentException.class, () -> new Buffer(config, recordFormatter)); - } - - { - Buffer.Config config = new Buffer.Config(); - config.setChunkRetentionSize(64 * 1024 * 1024); - config.setMaxBufferSize(64 * 1024 * 1024); - assertThrows(IllegalArgumentException.class, () -> new Buffer(config, recordFormatter)); - } - - { - Buffer.Config config = new Buffer.Config(); - config.setChunkExpandRatio(1.19f); - assertThrows(IllegalArgumentException.class, () -> new Buffer(config, recordFormatter)); - } - - { - Buffer.Config config = new Buffer.Config(); - config.setChunkRetentionTimeMillis(49); - assertThrows(IllegalArgumentException.class, () -> new Buffer(config, recordFormatter)); - } + Buffer.Config config = new Buffer.Config(); + config.setChunkRetentionTimeMillis(49); + assertThrows(IllegalArgumentException.class, () -> new Buffer(config, recordFormatter)); } + } } diff --git a/fluency-core/src/test/java/org/komamitsu/fluency/buffer/FileBackupTest.java b/fluency-core/src/test/java/org/komamitsu/fluency/buffer/FileBackupTest.java index 02261888..f4b88381 100644 --- a/fluency-core/src/test/java/org/komamitsu/fluency/buffer/FileBackupTest.java +++ b/fluency-core/src/test/java/org/komamitsu/fluency/buffer/FileBackupTest.java @@ -1,6 +1,8 @@ package org.komamitsu.fluency.buffer; -import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; import java.io.File; import java.io.IOException; @@ -13,10 +15,7 @@ import java.util.List; import java.util.Objects; import java.util.stream.Collectors; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; +import org.junit.jupiter.api.Test; class FileBackupTest { @@ -26,29 +25,36 @@ private void createTempFile(File dir, String filename, String content) throws IO tempfilePath.toFile().deleteOnExit(); } - private void assertSavedBuffer(FileBackup.SavedBuffer savedBuffer, Path expectedPath, byte[] expectedContent, String... expectedParams) { + private void assertSavedBuffer( + FileBackup.SavedBuffer savedBuffer, + Path expectedPath, + byte[] expectedContent, + String... expectedParams) { assertThat(savedBuffer.getPath()).isEqualTo(expectedPath); - savedBuffer.open((params, channel) -> { - assertThat(params.toArray()).isEqualTo(expectedParams); - try { - long size = channel.size(); - ByteBuffer buf = ByteBuffer.allocate((int) size); - channel.read(buf); - assertThat(buf.array()).isEqualTo(expectedContent); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + savedBuffer.open( + (params, channel) -> { + assertThat(params.toArray()).isEqualTo(expectedParams); + try { + long size = channel.size(); + ByteBuffer buf = ByteBuffer.allocate((int) size); + channel.read(buf); + assertThat(buf.array()).isEqualTo(expectedContent); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } - private void assertSavedFile(File savedFile, - String bufferFormatType, - String prefix, - long startNanos, - long endNanos, - String param1, - String param2, - byte[] expectedContent) throws IOException { + private void assertSavedFile( + File savedFile, + String bufferFormatType, + String prefix, + long startNanos, + long endNanos, + String param1, + String param2, + byte[] expectedContent) + throws IOException { String fileName = savedFile.toPath().getFileName().toString(); assertThat(fileName).endsWith(".buf"); @@ -78,37 +84,48 @@ void getSavedFiles_GivenSomeFiles_ShouldReturnThem() throws IOException { long nanoSeconds3 = System.nanoTime(); File backupDir = Files.createTempDirectory("test").toFile(); backupDir.deleteOnExit(); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("xmy_buf_type_my_prefix#param_a#param_b#%d.buf", System.nanoTime()), "ignored"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("xmy_buf_type_my_prefix#param_a#param_b#%d.buf", System.nanoTime()), "ignored"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("y_buf_type_my_prefix#param_a#param_b#%d.buf", System.nanoTime()), "ignored"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("my_buf_type_my_prefix#1paramA#1paramB#%d.buf", nanoSeconds1), "content1"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("my_buf_type_my_prefix#2param-a#2param-b#%d.buf", nanoSeconds2), "content2"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("my_buf_type_my_prefix#3param_a#3param_b#%d.buf", nanoSeconds3), "content3"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("my_buf_type_my_prefixz#param_a#param_b#%d.buf", System.nanoTime()), "ignored"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("my_buf_type_my_prefi#param_a#param_b#%d.buf", System.nanoTime()), "ignored"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("my_buf_type_my_prefix#param:a#param:b#%d.buf", System.nanoTime()), "ignored"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("my_buf_type_my_prefix#param_a#param_b#%d", System.nanoTime()), "ignored"); - createTempFile(backupDir, + createTempFile( + backupDir, String.format("my_buf_type_my_prefix#param_a#param_b#%d.buff", System.nanoTime()), "ignored"); Buffer buffer = mock(Buffer.class); @@ -116,22 +133,33 @@ void getSavedFiles_GivenSomeFiles_ShouldReturnThem() throws IOException { String prefix = "my_prefix"; FileBackup fileBackup = new FileBackup(backupDir, buffer, prefix); - List savedFiles = fileBackup.getSavedFiles().stream().sorted( - Comparator.comparing(FileBackup.SavedBuffer::getPath)).collect(Collectors.toList()); + List savedFiles = + fileBackup.getSavedFiles().stream() + .sorted(Comparator.comparing(FileBackup.SavedBuffer::getPath)) + .collect(Collectors.toList()); System.out.println(savedFiles); assertThat(savedFiles).size().isEqualTo(3); - assertSavedBuffer(savedFiles.get(0), - backupDir.toPath().resolve(String.format("my_buf_type_my_prefix#1paramA#1paramB#%d.buf", nanoSeconds1)), + assertSavedBuffer( + savedFiles.get(0), + backupDir + .toPath() + .resolve(String.format("my_buf_type_my_prefix#1paramA#1paramB#%d.buf", nanoSeconds1)), "content1".getBytes(StandardCharsets.UTF_8), "1paramA", "1paramB"); - assertSavedBuffer(savedFiles.get(1), - backupDir.toPath().resolve(String.format("my_buf_type_my_prefix#2param-a#2param-b#%d.buf", nanoSeconds2)), + assertSavedBuffer( + savedFiles.get(1), + backupDir + .toPath() + .resolve(String.format("my_buf_type_my_prefix#2param-a#2param-b#%d.buf", nanoSeconds2)), "content2".getBytes(StandardCharsets.UTF_8), "2param-a", "2param-b"); - assertSavedBuffer(savedFiles.get(2), - backupDir.toPath().resolve(String.format("my_buf_type_my_prefix#3param_a#3param_b#%d.buf", nanoSeconds3)), + assertSavedBuffer( + savedFiles.get(2), + backupDir + .toPath() + .resolve(String.format("my_buf_type_my_prefix#3param_a#3param_b#%d.buf", nanoSeconds3)), "content3".getBytes(StandardCharsets.UTF_8), "3param_a", "3param_b"); @@ -157,11 +185,13 @@ void saveBuffer() throws IOException { ByteBuffer.wrap("content3".getBytes(StandardCharsets.UTF_8))); long endNanos = System.nanoTime(); - List savedFiles = Arrays.stream(Objects.requireNonNull(backupDir.listFiles())) - .sorted(Comparator.comparing(File::toString)) - .collect(Collectors.toList()); + List savedFiles = + Arrays.stream(Objects.requireNonNull(backupDir.listFiles())) + .sorted(Comparator.comparing(File::toString)) + .collect(Collectors.toList()); assertThat(savedFiles).size().isEqualTo(3); - assertSavedFile(savedFiles.get(0), + assertSavedFile( + savedFiles.get(0), "my_buf_type", "my_prefix", startNanos, @@ -169,7 +199,8 @@ void saveBuffer() throws IOException { "1paramA", "1paramB", "content1".getBytes(StandardCharsets.UTF_8)); - assertSavedFile(savedFiles.get(1), + assertSavedFile( + savedFiles.get(1), "my_buf_type", "my_prefix", startNanos, @@ -177,7 +208,8 @@ void saveBuffer() throws IOException { "2param-a", "2param-b", "content2".getBytes(StandardCharsets.UTF_8)); - assertSavedFile(savedFiles.get(2), + assertSavedFile( + savedFiles.get(2), "my_buf_type", "my_prefix", startNanos, @@ -186,4 +218,4 @@ void saveBuffer() throws IOException { "3param_b", "content3".getBytes(StandardCharsets.UTF_8)); } -} \ No newline at end of file +} diff --git a/fluency-core/src/test/java/org/komamitsu/fluency/flusher/FlusherTest.java b/fluency-core/src/test/java/org/komamitsu/fluency/flusher/FlusherTest.java index d51dff21..85cca975 100644 --- a/fluency-core/src/test/java/org/komamitsu/fluency/flusher/FlusherTest.java +++ b/fluency-core/src/test/java/org/komamitsu/fluency/flusher/FlusherTest.java @@ -16,17 +16,6 @@ package org.komamitsu.fluency.flusher; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.JsonRecordFormatter; -import org.komamitsu.fluency.buffer.Buffer; -import org.komamitsu.fluency.ingester.Ingester; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.TimeUnit; - import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.eq; @@ -37,117 +26,122 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -class FlusherTest -{ - private Ingester ingester; - private Buffer.Config bufferConfig; - private Flusher.Config flusherConfig; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.JsonRecordFormatter; +import org.komamitsu.fluency.buffer.Buffer; +import org.komamitsu.fluency.ingester.Ingester; + +class FlusherTest { + private Ingester ingester; + private Buffer.Config bufferConfig; + private Flusher.Config flusherConfig; - @BeforeEach - void setUp() - { - ingester = mock(Ingester.class); + @BeforeEach + void setUp() { + ingester = mock(Ingester.class); - bufferConfig = new Buffer.Config(); - flusherConfig = new Flusher.Config(); - } + bufferConfig = new Buffer.Config(); + flusherConfig = new Flusher.Config(); + } - @Test - void testAsyncFlusher() - throws IOException, InterruptedException - { - flusherConfig.setFlushAttemptIntervalMillis(500); + @Test + void testAsyncFlusher() throws IOException, InterruptedException { + flusherConfig.setFlushAttemptIntervalMillis(500); + + Buffer buffer = spy(new Buffer(bufferConfig, new JsonRecordFormatter())); + Flusher flusher = new Flusher(flusherConfig, buffer, ingester); - Buffer buffer = spy(new Buffer(bufferConfig, new JsonRecordFormatter())); - Flusher flusher = new Flusher(flusherConfig, buffer, ingester); + verify(buffer, times(0)).flush(eq(ingester), anyBoolean()); - verify(buffer, times(0)).flush(eq(ingester), anyBoolean()); + verify(buffer, times(0)).flush(eq(ingester), anyBoolean()); - verify(buffer, times(0)).flush(eq(ingester), anyBoolean()); + flusher.flush(); + TimeUnit.MILLISECONDS.sleep(50); + verify(buffer, times(0)).flush(eq(ingester), eq(false)); + verify(buffer, times(1)).flush(eq(ingester), eq(true)); - flusher.flush(); - TimeUnit.MILLISECONDS.sleep(50); - verify(buffer, times(0)).flush(eq(ingester), eq(false)); - verify(buffer, times(1)).flush(eq(ingester), eq(true)); + TimeUnit.SECONDS.sleep(1); + verify(buffer, atLeast(1)).flush(eq(ingester), eq(false)); + verify(buffer, atMost(3)).flush(eq(ingester), eq(false)); + verify(buffer, times(1)).flush(eq(ingester), eq(true)); - TimeUnit.SECONDS.sleep(1); - verify(buffer, atLeast(1)).flush(eq(ingester), eq(false)); - verify(buffer, atMost(3)).flush(eq(ingester), eq(false)); - verify(buffer, times(1)).flush(eq(ingester), eq(true)); + verify(buffer, times(0)).close(); - verify(buffer, times(0)).close(); + flusher.close(); + verify(buffer, times(1)).close(); - flusher.close(); - verify(buffer, times(1)).close(); + verify(buffer, atLeast(2)).flush(eq(ingester), eq(false)); + verify(buffer, atMost(3)).flush(eq(ingester), eq(false)); - verify(buffer, atLeast(2)).flush(eq(ingester), eq(false)); - verify(buffer, atMost(3)).flush(eq(ingester), eq(false)); + verify(buffer, atLeast(2)).flush(eq(ingester), eq(true)); + verify(buffer, atMost(3)).flush(eq(ingester), eq(true)); + } - verify(buffer, atLeast(2)).flush(eq(ingester), eq(true)); - verify(buffer, atMost(3)).flush(eq(ingester), eq(true)); + @Test + void validateConfig() { + Buffer buffer = spy(new Buffer(bufferConfig, new JsonRecordFormatter())); + + { + Flusher.Config config = new Flusher.Config(); + config.setFlushAttemptIntervalMillis(19); + assertThrows(IllegalArgumentException.class, () -> new Flusher(config, buffer, ingester)); } - @Test - void validateConfig() { - Buffer buffer = spy(new Buffer(bufferConfig, new JsonRecordFormatter())); - - { - Flusher.Config config = new Flusher.Config(); - config.setFlushAttemptIntervalMillis(19); - assertThrows(IllegalArgumentException.class, () -> new Flusher(config, buffer, ingester)); - } - - { - Flusher.Config config = new Flusher.Config(); - config.setFlushAttemptIntervalMillis(2001); - assertThrows(IllegalArgumentException.class, () -> new Flusher(config, buffer, ingester)); - } - - { - Flusher.Config config = new Flusher.Config(); - config.setWaitUntilBufferFlushed(0); - assertThrows(IllegalArgumentException.class, () -> new Flusher(config, buffer, ingester)); - } - - { - Flusher.Config config = new Flusher.Config(); - config.setWaitUntilTerminated(0); - assertThrows(IllegalArgumentException.class, () -> new Flusher(config, buffer, ingester)); - } + Flusher.Config config = new Flusher.Config(); + config.setFlushAttemptIntervalMillis(2001); + assertThrows(IllegalArgumentException.class, () -> new Flusher(config, buffer, ingester)); } - @Test - void queueSizeShouldNotExceedLimit() throws Exception { - flusherConfig.setFlushAttemptIntervalMillis(200); - - Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); - Flusher flusher = new Flusher(flusherConfig, buffer, ingester); - - BlockingQueue eventQueue = (BlockingQueue) extractValue(flusher, "eventQueue"); - - int nonZeroCounts = 0; - for (int index = 0; index < 10_000; index++) { - flusher.flush(); - if(!eventQueue.isEmpty()) { - nonZeroCounts++; - } - // The eventQueue will always have less that 16 elements (the default max size) - assertTrue(eventQueue.size() <= 16); - } - // The eventQueue will be non empty at least a couple of times - assertTrue(nonZeroCounts > 0); - - // Wait for sufficiently long (amount of time > queue poll wait) and the queue should be polled completely to become empty - Thread.sleep(1000); - assertEquals(0, eventQueue.size()); + Flusher.Config config = new Flusher.Config(); + config.setWaitUntilBufferFlushed(0); + assertThrows(IllegalArgumentException.class, () -> new Flusher(config, buffer, ingester)); } - private Object extractValue(Object object, String fieldName) throws Exception { - Field field = object.getClass().getDeclaredField(fieldName); - field.setAccessible(true); - return field.get(object); + Flusher.Config config = new Flusher.Config(); + config.setWaitUntilTerminated(0); + assertThrows(IllegalArgumentException.class, () -> new Flusher(config, buffer, ingester)); + } + } + + @Test + void queueSizeShouldNotExceedLimit() throws Exception { + flusherConfig.setFlushAttemptIntervalMillis(200); + + Buffer buffer = new Buffer(bufferConfig, new JsonRecordFormatter()); + Flusher flusher = new Flusher(flusherConfig, buffer, ingester); + + BlockingQueue eventQueue = + (BlockingQueue) extractValue(flusher, "eventQueue"); + + int nonZeroCounts = 0; + for (int index = 0; index < 10_000; index++) { + flusher.flush(); + if (!eventQueue.isEmpty()) { + nonZeroCounts++; + } + // The eventQueue will always have less that 16 elements (the default max size) + assertTrue(eventQueue.size() <= 16); } + // The eventQueue will be non empty at least a couple of times + assertTrue(nonZeroCounts > 0); + + // Wait for sufficiently long (amount of time > queue poll wait) and the queue should be polled + // completely to become empty + Thread.sleep(1000); + assertEquals(0, eventQueue.size()); + } + + private Object extractValue(Object object, String fieldName) throws Exception { + Field field = object.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + return field.get(object); + } } diff --git a/fluency-core/src/test/java/org/komamitsu/fluency/recordformat/MessagePackRecordFormatterTest.java b/fluency-core/src/test/java/org/komamitsu/fluency/recordformat/MessagePackRecordFormatterTest.java index b3065f65..0b6b76da 100644 --- a/fluency-core/src/test/java/org/komamitsu/fluency/recordformat/MessagePackRecordFormatterTest.java +++ b/fluency-core/src/test/java/org/komamitsu/fluency/recordformat/MessagePackRecordFormatterTest.java @@ -16,9 +16,14 @@ package org.komamitsu.fluency.recordformat; +import static org.junit.jupiter.api.Assertions.assertEquals; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.msgpack.core.MessagePack; @@ -28,126 +33,117 @@ import org.msgpack.value.Value; import org.msgpack.value.ValueFactory; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; +class MessagePackRecordFormatterTest { + private static final String TAG = "foodb.bartbl"; + private static final long FUTURE_EPOCH = 4294967296L; + private static final Map RECORD_0 = + ImmutableMap.of("name", "first", "age", 42, "email", "hello@world.com"); + private static final Map RECORD_1 = + ImmutableMap.of("name", "second", "age", 55, "time", FUTURE_EPOCH, "comment", "zzzzzz"); + private static final Map RECORD_2 = + ImmutableMap.of("job", "knight", "name", "third", "age", 99); + private static final StringValue KEY_TIME = ValueFactory.newString("time"); + private static final StringValue KEY_NAME = ValueFactory.newString("name"); + private static final StringValue KEY_AGE = ValueFactory.newString("age"); + private static final StringValue KEY_EMAIL = ValueFactory.newString("email"); + private static final StringValue KEY_COMMENT = ValueFactory.newString("comment"); + private static final StringValue KEY_JOB = ValueFactory.newString("job"); + private MessagePackRecordFormatter recordFormatter; -class MessagePackRecordFormatterTest -{ - private static final String TAG = "foodb.bartbl"; - private static final long FUTURE_EPOCH = 4294967296L; - private static final Map RECORD_0 = ImmutableMap.of("name", "first", "age", 42, "email", "hello@world.com"); - private static final Map RECORD_1 = ImmutableMap.of("name", "second", "age", 55, "time", FUTURE_EPOCH, "comment", "zzzzzz"); - private static final Map RECORD_2 = ImmutableMap.of("job", "knight", "name", "third", "age", 99); - private static final StringValue KEY_TIME = ValueFactory.newString("time"); - private static final StringValue KEY_NAME = ValueFactory.newString("name"); - private static final StringValue KEY_AGE = ValueFactory.newString("age"); - private static final StringValue KEY_EMAIL = ValueFactory.newString("email"); - private static final StringValue KEY_COMMENT = ValueFactory.newString("comment"); - private static final StringValue KEY_JOB = ValueFactory.newString("job"); - private MessagePackRecordFormatter recordFormatter; + @BeforeEach + void setUp() { + recordFormatter = new MessagePackRecordFormatter(new MessagePackRecordFormatter.Config()); + } - @BeforeEach - void setUp() - { - recordFormatter = new MessagePackRecordFormatter(new MessagePackRecordFormatter.Config()); + private void assertRecord0(byte[] formatted, long expectedTime) throws IOException { + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(formatted)) { + Map map = unpacker.unpackValue().asMapValue().map(); + assertEquals(4, map.size()); + assertEquals(expectedTime, map.get(KEY_TIME).asNumberValue().toLong()); + assertEquals("first", map.get(KEY_NAME).asStringValue().asString()); + assertEquals(42, map.get(KEY_AGE).asNumberValue().toInt()); + assertEquals("hello@world.com", map.get(KEY_EMAIL).asStringValue().asString()); } + } - private void assertRecord0(byte[] formatted, long expectedTime) - throws IOException - { - try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(formatted)) { - Map map = unpacker.unpackValue().asMapValue().map(); - assertEquals(4, map.size()); - assertEquals(expectedTime, map.get(KEY_TIME).asNumberValue().toLong()); - assertEquals("first", map.get(KEY_NAME).asStringValue().asString()); - assertEquals(42, map.get(KEY_AGE).asNumberValue().toInt()); - assertEquals("hello@world.com", map.get(KEY_EMAIL).asStringValue().asString()); - } + private void assertRecord1(byte[] formatted, long expectedTime) throws IOException { + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(formatted)) { + Map map = unpacker.unpackValue().asMapValue().map(); + assertEquals(4, map.size()); + assertEquals(expectedTime, map.get(KEY_TIME).asNumberValue().toLong()); + assertEquals("second", map.get(KEY_NAME).asStringValue().asString()); + assertEquals(55, map.get(KEY_AGE).asNumberValue().toInt()); + assertEquals("zzzzzz", map.get(KEY_COMMENT).asStringValue().asString()); } + } - private void assertRecord1(byte[] formatted, long expectedTime) - throws IOException - { - try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(formatted)) { - Map map = unpacker.unpackValue().asMapValue().map(); - assertEquals(4, map.size()); - assertEquals(expectedTime, map.get(KEY_TIME).asNumberValue().toLong()); - assertEquals("second", map.get(KEY_NAME).asStringValue().asString()); - assertEquals(55, map.get(KEY_AGE).asNumberValue().toInt()); - assertEquals("zzzzzz", map.get(KEY_COMMENT).asStringValue().asString()); - } + private void assertRecord2(byte[] formatted, long expectedTime) throws IOException { + try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(formatted)) { + Map map = unpacker.unpackValue().asMapValue().map(); + assertEquals(4, map.size()); + assertEquals(expectedTime, map.get(KEY_TIME).asNumberValue().toLong()); + assertEquals("third", map.get(KEY_NAME).asStringValue().asString()); + assertEquals(99, map.get(KEY_AGE).asNumberValue().toInt()); + assertEquals("knight", map.get(KEY_JOB).asStringValue().asString()); } + } - private void assertRecord2(byte[] formatted, long expectedTime) - throws IOException - { - try (MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(formatted)) { - Map map = unpacker.unpackValue().asMapValue().map(); - assertEquals(4, map.size()); - assertEquals(expectedTime, map.get(KEY_TIME).asNumberValue().toLong()); - assertEquals("third", map.get(KEY_NAME).asStringValue().asString()); - assertEquals(99, map.get(KEY_AGE).asNumberValue().toInt()); - assertEquals("knight", map.get(KEY_JOB).asStringValue().asString()); - } - } + @Test + void format() throws IOException { + long now = System.currentTimeMillis() / 1000; + assertRecord0(recordFormatter.format(TAG, now, RECORD_0), now); + assertRecord1(recordFormatter.format(TAG, now, RECORD_1), FUTURE_EPOCH); + assertRecord2(recordFormatter.format(TAG, now, RECORD_2), now); + } - @Test - void format() - throws IOException + @Test + void formatFromMessagePackBytes() throws IOException { + long now = System.currentTimeMillis() / 1000; + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); { - long now = System.currentTimeMillis() / 1000; - assertRecord0(recordFormatter.format(TAG, now, RECORD_0), now); - assertRecord1(recordFormatter.format(TAG, now, RECORD_1), FUTURE_EPOCH); - assertRecord2(recordFormatter.format(TAG, now, RECORD_2), now); + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_0); + assertRecord0(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now); } - - @Test - void formatFromMessagePackBytes() - throws IOException { - long now = System.currentTimeMillis() / 1000; - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_0); - assertRecord0(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now); - } - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_1); - assertRecord1(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), FUTURE_EPOCH); - } - { - byte[] bytes = objectMapper.writeValueAsBytes(RECORD_2); - assertRecord2(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now); - } + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_1); + assertRecord1( + recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), FUTURE_EPOCH); } - - private ByteBuffer convertMapToMessagePackByteBuffer(Map record, boolean isDirect) - throws JsonProcessingException { - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - byte[] bytes = objectMapper.writeValueAsBytes(record); - ByteBuffer byteBuffer; - if (isDirect) { - byteBuffer = ByteBuffer.allocateDirect(bytes.length); - } - else { - byteBuffer = ByteBuffer.allocate(bytes.length); - } - byteBuffer.put(bytes); - byteBuffer.flip(); - return byteBuffer; + byte[] bytes = objectMapper.writeValueAsBytes(RECORD_2); + assertRecord2(recordFormatter.formatFromMessagePack(TAG, now, bytes, 0, bytes.length), now); } + } - @Test - void formatFromMessagePackByteBuffer() - throws IOException - { - long now = System.currentTimeMillis() / 1000; - assertRecord0(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_0, false)), now); - assertRecord1(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_1, true)), FUTURE_EPOCH); - assertRecord2(recordFormatter.formatFromMessagePack(TAG, now, convertMapToMessagePackByteBuffer(RECORD_2, true)), now); + private ByteBuffer convertMapToMessagePackByteBuffer(Map record, boolean isDirect) + throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + byte[] bytes = objectMapper.writeValueAsBytes(record); + ByteBuffer byteBuffer; + if (isDirect) { + byteBuffer = ByteBuffer.allocateDirect(bytes.length); + } else { + byteBuffer = ByteBuffer.allocate(bytes.length); } + byteBuffer.put(bytes); + byteBuffer.flip(); + return byteBuffer; + } + + @Test + void formatFromMessagePackByteBuffer() throws IOException { + long now = System.currentTimeMillis() / 1000; + assertRecord0( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_0, false)), + now); + assertRecord1( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_1, true)), + FUTURE_EPOCH); + assertRecord2( + recordFormatter.formatFromMessagePack( + TAG, now, convertMapToMessagePackByteBuffer(RECORD_2, true)), + now); + } } diff --git a/fluency-core/src/test/java/org/komamitsu/fluency/validation/ValidatableTest.java b/fluency-core/src/test/java/org/komamitsu/fluency/validation/ValidatableTest.java index 3d210189..881dd3e7 100644 --- a/fluency-core/src/test/java/org/komamitsu/fluency/validation/ValidatableTest.java +++ b/fluency-core/src/test/java/org/komamitsu/fluency/validation/ValidatableTest.java @@ -16,147 +16,132 @@ package org.komamitsu.fluency.validation; +import static org.junit.jupiter.api.Assertions.assertThrows; + import org.junit.jupiter.api.Test; import org.komamitsu.fluency.validation.annotation.DecimalMax; import org.komamitsu.fluency.validation.annotation.DecimalMin; import org.komamitsu.fluency.validation.annotation.Max; import org.komamitsu.fluency.validation.annotation.Min; -import static org.junit.jupiter.api.Assertions.assertThrows; +class ValidatableTest { + private static class MaxTest implements Validatable { + @Max(42) + private final int i; + + @Max(Integer.MAX_VALUE) + private final Long l; + + @Max(value = 42, inclusive = false) + private final int exclusive; -class ValidatableTest -{ - private static class MaxTest - implements Validatable - { - @Max(42) - private final int i; - - @Max(Integer.MAX_VALUE) - private final Long l; - - @Max(value = 42, inclusive = false) - private final int exclusive; - - public MaxTest(int i, Long l, int exclusive) - { - this.i = i; - this.l = l; - this.exclusive = exclusive; - } + public MaxTest(int i, Long l, int exclusive) { + this.i = i; + this.l = l; + this.exclusive = exclusive; } + } - private static class MinTest - implements Validatable - { - @Min(42) - private final int i; + private static class MinTest implements Validatable { + @Min(42) + private final int i; - @Min(Integer.MIN_VALUE) - private final Long l; + @Min(Integer.MIN_VALUE) + private final Long l; - @Min(value = 42, inclusive = false) - private final int exclusive; + @Min(value = 42, inclusive = false) + private final int exclusive; - public MinTest(int i, Long l, int exclusive) - { - this.i = i; - this.l = l; - this.exclusive = exclusive; - } + public MinTest(int i, Long l, int exclusive) { + this.i = i; + this.l = l; + this.exclusive = exclusive; } + } - private static class DecimalMaxTest - implements Validatable - { - @DecimalMax("3.14") - private final float f; + private static class DecimalMaxTest implements Validatable { + @DecimalMax("3.14") + private final float f; - @DecimalMax(value = "3.14", inclusive = false) - private final Double exclusive; + @DecimalMax(value = "3.14", inclusive = false) + private final Double exclusive; - public DecimalMaxTest(float f, Double exclusive) - { - this.f = f; - this.exclusive = exclusive; - } + public DecimalMaxTest(float f, Double exclusive) { + this.f = f; + this.exclusive = exclusive; } + } - private static class DecimalMinTest - implements Validatable - { - @DecimalMin("3.14") - private final float f; + private static class DecimalMinTest implements Validatable { + @DecimalMin("3.14") + private final float f; - @DecimalMin(value = "3.14", inclusive = false) - private final Double exclusive; + @DecimalMin(value = "3.14", inclusive = false) + private final Double exclusive; - public DecimalMinTest(float f, Double exclusive) - { - this.f = f; - this.exclusive = exclusive; - } + public DecimalMinTest(float f, Double exclusive) { + this.f = f; + this.exclusive = exclusive; } + } - @Test - void validateMax() - { - new MaxTest(42, (long) Integer.MAX_VALUE, 41).validate(); + @Test + void validateMax() { + new MaxTest(42, (long) Integer.MAX_VALUE, 41).validate(); - assertThrows(IllegalArgumentException.class, - () -> new MaxTest(43, (long) Integer.MAX_VALUE, 41).validate()); + assertThrows( + IllegalArgumentException.class, + () -> new MaxTest(43, (long) Integer.MAX_VALUE, 41).validate()); - assertThrows(IllegalArgumentException.class, - () -> new MaxTest(42, (long) Integer.MAX_VALUE + 1, 41).validate()); + assertThrows( + IllegalArgumentException.class, + () -> new MaxTest(42, (long) Integer.MAX_VALUE + 1, 41).validate()); - assertThrows(IllegalArgumentException.class, - () -> new MaxTest(42, (long) Integer.MAX_VALUE, 42).validate()); + assertThrows( + IllegalArgumentException.class, + () -> new MaxTest(42, (long) Integer.MAX_VALUE, 42).validate()); - new MaxTest(42, null, 41).validate(); - } + new MaxTest(42, null, 41).validate(); + } - @Test - void validateMin() - { - new MinTest(42, (long) Integer.MIN_VALUE, 43).validate(); + @Test + void validateMin() { + new MinTest(42, (long) Integer.MIN_VALUE, 43).validate(); - assertThrows(IllegalArgumentException.class, - () -> new MinTest(41, (long) Integer.MIN_VALUE, 43).validate()); + assertThrows( + IllegalArgumentException.class, + () -> new MinTest(41, (long) Integer.MIN_VALUE, 43).validate()); - assertThrows(IllegalArgumentException.class, - () -> new MinTest(42, (long) Integer.MIN_VALUE - 1, 43).validate()); + assertThrows( + IllegalArgumentException.class, + () -> new MinTest(42, (long) Integer.MIN_VALUE - 1, 43).validate()); - assertThrows(IllegalArgumentException.class, - () -> new MinTest(42, (long) Integer.MIN_VALUE, 42).validate()); + assertThrows( + IllegalArgumentException.class, + () -> new MinTest(42, (long) Integer.MIN_VALUE, 42).validate()); - new MinTest(42, null, 43).validate(); - } + new MinTest(42, null, 43).validate(); + } - @Test - void validateDecimalMax() - { - new DecimalMaxTest(3.14f, 3.13).validate(); + @Test + void validateDecimalMax() { + new DecimalMaxTest(3.14f, 3.13).validate(); - assertThrows(IllegalArgumentException.class, - () -> new DecimalMaxTest(3.15f, 3.13).validate()); + assertThrows(IllegalArgumentException.class, () -> new DecimalMaxTest(3.15f, 3.13).validate()); - assertThrows(IllegalArgumentException.class, - () -> new DecimalMaxTest(3.14f, 3.14).validate()); + assertThrows(IllegalArgumentException.class, () -> new DecimalMaxTest(3.14f, 3.14).validate()); - new DecimalMaxTest(3.14f, null).validate(); - } + new DecimalMaxTest(3.14f, null).validate(); + } - @Test - void validateDecimalMin() - { - new DecimalMinTest(3.14f, 3.15).validate(); + @Test + void validateDecimalMin() { + new DecimalMinTest(3.14f, 3.15).validate(); - assertThrows(IllegalArgumentException.class, - () -> new DecimalMinTest(3.13f, 3.15).validate()); + assertThrows(IllegalArgumentException.class, () -> new DecimalMinTest(3.13f, 3.15).validate()); - assertThrows(IllegalArgumentException.class, - () -> new DecimalMinTest(3.14f, 3.14).validate()); + assertThrows(IllegalArgumentException.class, () -> new DecimalMinTest(3.14f, 3.14).validate()); - new DecimalMinTest(3.14f, null).validate(); - } -} \ No newline at end of file + new DecimalMinTest(3.14f, null).validate(); + } +} diff --git a/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/FluencyExtBuilderForFluentd.java b/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/FluencyExtBuilderForFluentd.java index f74beb84..e622fd6f 100644 --- a/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/FluencyExtBuilderForFluentd.java +++ b/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/FluencyExtBuilderForFluentd.java @@ -16,48 +16,37 @@ package org.komamitsu.fluency.fluentd; +import java.nio.file.Path; import org.komamitsu.fluency.Fluency; import org.komamitsu.fluency.fluentd.ingester.sender.FluentdSender; -import org.komamitsu.fluency.fluentd.ingester.sender.MultiSender; import org.komamitsu.fluency.fluentd.ingester.sender.UnixSocketSender; import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.UnixSocketHeartbeater; -import java.net.UnixDomainSocketAddress; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; +public class FluencyExtBuilderForFluentd extends FluencyBuilderForFluentd { + public Fluency build(Path socketPath) { + return buildFromIngester(recordFormatter, buildIngester(createBaseSender(socketPath, false))); + } -public class FluencyExtBuilderForFluentd - extends FluencyBuilderForFluentd -{ - public Fluency build(Path socketPath) - { - return buildFromIngester( - recordFormatter, - buildIngester(createBaseSender(socketPath, false))); + private FluentdSender createBaseSender(Path path, boolean withHeartBeater) { + UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); + FailureDetector failureDetector = null; + if (path != null) { + senderConfig.setPath(path.toAbsolutePath()); } - - private FluentdSender createBaseSender(Path path, boolean withHeartBeater) - { - UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); - FailureDetector failureDetector = null; - if (path != null) { - senderConfig.setPath(path.toAbsolutePath()); - } - if (withHeartBeater) { - UnixSocketHeartbeater.Config hbConfig = new UnixSocketHeartbeater.Config(); - hbConfig.setPath(path.toAbsolutePath()); - UnixSocketHeartbeater heartbeater = new UnixSocketHeartbeater(hbConfig); - failureDetector = new FailureDetector(new PhiAccrualFailureDetectStrategy(), heartbeater); - } - if (connectionTimeoutMilli != null) { - senderConfig.setConnectionTimeoutMilli(connectionTimeoutMilli); - } - if (readTimeoutMilli != null) { - senderConfig.setReadTimeoutMilli(readTimeoutMilli); - } - return new UnixSocketSender(senderConfig, failureDetector); + if (withHeartBeater) { + UnixSocketHeartbeater.Config hbConfig = new UnixSocketHeartbeater.Config(); + hbConfig.setPath(path.toAbsolutePath()); + UnixSocketHeartbeater heartbeater = new UnixSocketHeartbeater(hbConfig); + failureDetector = new FailureDetector(new PhiAccrualFailureDetectStrategy(), heartbeater); + } + if (connectionTimeoutMilli != null) { + senderConfig.setConnectionTimeoutMilli(connectionTimeoutMilli); + } + if (readTimeoutMilli != null) { + senderConfig.setReadTimeoutMilli(readTimeoutMilli); } + return new UnixSocketSender(senderConfig, failureDetector); + } } diff --git a/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/UnixSocketSender.java b/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/UnixSocketSender.java index 30bb817f..e8696f53 100644 --- a/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/UnixSocketSender.java +++ b/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/UnixSocketSender.java @@ -16,11 +16,6 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; -import org.komamitsu.fluency.validation.Validatable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.SocketException; import java.net.UnixDomainSocketAddress; @@ -29,113 +24,92 @@ import java.nio.file.Path; import java.util.List; import java.util.concurrent.atomic.AtomicReference; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; +import org.komamitsu.fluency.validation.Validatable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class UnixSocketSender - extends NetworkSender -{ - private static final Logger LOG = LoggerFactory.getLogger(UnixSocketSender.class); - private final AtomicReference channel = new AtomicReference<>(); - private final Config config; - - public UnixSocketSender() - { - this(new Config()); - } - - public UnixSocketSender(Config config) - { - this(config, null); +public class UnixSocketSender extends NetworkSender { + private static final Logger LOG = LoggerFactory.getLogger(UnixSocketSender.class); + private final AtomicReference channel = new AtomicReference<>(); + private final Config config; + + public UnixSocketSender() { + this(new Config()); + } + + public UnixSocketSender(Config config) { + this(config, null); + } + + public UnixSocketSender(FailureDetector failureDetector) { + this(new Config(), failureDetector); + } + + public UnixSocketSender(Config config, FailureDetector failureDetector) { + super(config, failureDetector); + config.validateValues(); + this.config = config; + } + + @Override + protected SocketChannel getOrCreateSocketInternal() throws IOException { + if (channel.get() == null) { + UnixDomainSocketAddress socketAddress = UnixDomainSocketAddress.of(config.getPath()); + channel.set(SocketChannel.open(socketAddress)); } - - public UnixSocketSender(FailureDetector failureDetector) - { - this(new Config(), failureDetector); + return channel.get(); + } + + public Path getPath() { + return config.getPath(); + } + + @Override + protected void sendBuffers(SocketChannel socketChannel, List buffers) + throws IOException { + socketChannel.write(buffers.toArray(new ByteBuffer[0])); + } + + @Override + protected void recvResponse(SocketChannel socketChannel, ByteBuffer buffer) throws IOException { + int read = socketChannel.read(buffer); + if (read < 0) { + throw new SocketException("The connection is disconnected by the peer"); } + } - public UnixSocketSender(Config config, FailureDetector failureDetector) - { - super(config, failureDetector); - config.validateValues(); - this.config = config; + @Override + protected void closeSocket() throws IOException { + SocketChannel existingSocketChannel; + if ((existingSocketChannel = channel.getAndSet(null)) != null) { + existingSocketChannel.close(); } + } - @Override - protected SocketChannel getOrCreateSocketInternal() - throws IOException - { - if (channel.get() == null) { - UnixDomainSocketAddress socketAddress = UnixDomainSocketAddress.of(config.getPath()); - channel.set(SocketChannel.open(socketAddress)); - } - return channel.get(); - } + @Override + public String toString() { + return "UnixSocketSender{" + "config=" + config + "} " + super.toString(); + } - public Path getPath() - { - return config.getPath(); - } + public static class Config extends NetworkSender.Config implements Validatable { + private Path path; - @Override - protected void sendBuffers(SocketChannel socketChannel, List buffers) - throws IOException - { - socketChannel.write(buffers.toArray(new ByteBuffer[0])); + void validateValues() { + validate(); } - @Override - protected void recvResponse(SocketChannel socketChannel, ByteBuffer buffer) - throws IOException - { - int read = socketChannel.read(buffer); - if (read < 0) { - throw new SocketException("The connection is disconnected by the peer"); - } + public Path getPath() { + return path; } - @Override - protected void closeSocket() - throws IOException - { - SocketChannel existingSocketChannel; - if ((existingSocketChannel = channel.getAndSet(null)) != null) { - existingSocketChannel.close(); - } + public void setPath(Path path) { + this.path = path; } @Override - public String toString() - { - return "UnixSocketSender{" + - "config=" + config + - "} " + super.toString(); - } - - public static class Config - extends NetworkSender.Config - implements Validatable - { - private Path path; - - void validateValues() - { - validate(); - } - - public Path getPath() - { - return path; - } - - public void setPath(Path path) - { - this.path = path; - } - - @Override - public String toString() { - return "Config{" + - "path=" + path + - "} " + super.toString(); - } + public String toString() { + return "Config{" + "path=" + path + "} " + super.toString(); } + } } diff --git a/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UnixSocketHeartbeater.java b/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UnixSocketHeartbeater.java index 66dddbb8..5efdfe1b 100644 --- a/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UnixSocketHeartbeater.java +++ b/fluency-fluentd-ext/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UnixSocketHeartbeater.java @@ -16,68 +16,53 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.UnixDomainSocketAddress; import java.nio.channels.SocketChannel; import java.nio.file.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class UnixSocketHeartbeater - extends Heartbeater -{ - private static final Logger LOG = LoggerFactory.getLogger(UnixSocketHeartbeater.class); - private final Config config; +public class UnixSocketHeartbeater extends Heartbeater { + private static final Logger LOG = LoggerFactory.getLogger(UnixSocketHeartbeater.class); + private final Config config; - public UnixSocketHeartbeater() - { - this(new Config()); - } + public UnixSocketHeartbeater() { + this(new Config()); + } - public UnixSocketHeartbeater(final Config config) - { - super(config); - this.config = config; - } + public UnixSocketHeartbeater(final Config config) { + super(config); + this.config = config; + } - @Override - protected void invoke() - throws IOException - { - try (SocketChannel socketChannel = SocketChannel.open(UnixDomainSocketAddress.of(config.getPath()))) { - LOG.trace("UnixSocketHeartbeat: {}", socketChannel); - pong(); - } + @Override + protected void invoke() throws IOException { + try (SocketChannel socketChannel = + SocketChannel.open(UnixDomainSocketAddress.of(config.getPath()))) { + LOG.trace("UnixSocketHeartbeat: {}", socketChannel); + pong(); } + } - public Path getPath() - { - return config.getPath(); - } + public Path getPath() { + return config.getPath(); + } - @Override - public String toString() - { - return "UnixSocketHeartbeater{" + - "config=" + config + - "} " + super.toString(); - } + @Override + public String toString() { + return "UnixSocketHeartbeater{" + "config=" + config + "} " + super.toString(); + } - public static class Config - extends Heartbeater.Config - { - private Path path; + public static class Config extends Heartbeater.Config { + private Path path; - public Path getPath() - { - return path; - } + public Path getPath() { + return path; + } - public void setPath(Path path) - { - this.path = path; - } + public void setPath(Path path) { + this.path = path; } + } } - diff --git a/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/FluencyExtBuilderForFluentdTest.java b/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/FluencyExtBuilderForFluentdTest.java index 715062a4..7807d008 100644 --- a/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/FluencyExtBuilderForFluentdTest.java +++ b/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/FluencyExtBuilderForFluentdTest.java @@ -1,5 +1,10 @@ package org.komamitsu.fluency.fluentd; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import org.junit.jupiter.api.Test; import org.komamitsu.fluency.Fluency; import org.komamitsu.fluency.buffer.Buffer; @@ -12,83 +17,73 @@ import org.komamitsu.fluency.fluentd.ingester.sender.retry.ExponentialBackOffRetryStrategy; import org.komamitsu.fluency.flusher.Flusher; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; +class FluencyExtBuilderForFluentdTest { + // These assertMethods are copied from FluencyBuilderForFluentdTest + private void assertBuffer(Buffer buffer) { + assertThat(buffer.getMaxBufferSize()).isEqualTo(512 * 1024 * 1024L); + assertThat(buffer.getFileBackupDir()).isNull(); + assertThat(buffer.bufferFormatType()).isEqualTo("packed_forward"); + assertThat(buffer.getChunkExpandRatio()).isEqualTo(2f); + assertThat(buffer.getChunkRetentionSize()).isEqualTo(4 * 1024 * 1024); + assertThat(buffer.getChunkInitialSize()).isEqualTo(1 * 1024 * 1024); + assertThat(buffer.getChunkRetentionTimeMillis()).isEqualTo(1000); + assertThat(buffer.getJvmHeapBufferMode()).isFalse(); + } -import static org.assertj.core.api.Assertions.assertThat; + private void assertFlusher(Flusher flusher) { + assertThat(flusher.isTerminated()).isFalse(); + assertThat(flusher.getFlushAttemptIntervalMillis()).isEqualTo(600); + assertThat(flusher.getWaitUntilBufferFlushed()).isEqualTo(60); + assertThat(flusher.getWaitUntilTerminated()).isEqualTo(60); + } -class FluencyExtBuilderForFluentdTest -{ - // These assertMethods are copied from FluencyBuilderForFluentdTest - private void assertBuffer(Buffer buffer) - { - assertThat(buffer.getMaxBufferSize()).isEqualTo(512 * 1024 * 1024L); - assertThat(buffer.getFileBackupDir()).isNull(); - assertThat(buffer.bufferFormatType()).isEqualTo("packed_forward"); - assertThat(buffer.getChunkExpandRatio()).isEqualTo(2f); - assertThat(buffer.getChunkRetentionSize()).isEqualTo(4 * 1024 * 1024); - assertThat(buffer.getChunkInitialSize()).isEqualTo(1 * 1024 * 1024); - assertThat(buffer.getChunkRetentionTimeMillis()).isEqualTo(1000); - assertThat(buffer.getJvmHeapBufferMode()).isFalse(); - } + private void assertDefaultRetryableSender( + RetryableSender sender, Class expectedBaseClass) { + assertThat(sender.getRetryStrategy()).isInstanceOf(ExponentialBackOffRetryStrategy.class); + ExponentialBackOffRetryStrategy retryStrategy = + (ExponentialBackOffRetryStrategy) sender.getRetryStrategy(); + assertThat(retryStrategy.getMaxRetryCount()).isEqualTo(7); + assertThat(retryStrategy.getBaseIntervalMillis()).isEqualTo(400); + assertThat(sender.getBaseSender()).isInstanceOf(expectedBaseClass); + } - private void assertFlusher(Flusher flusher) - { - assertThat(flusher.isTerminated()).isFalse(); - assertThat(flusher.getFlushAttemptIntervalMillis()).isEqualTo(600); - assertThat(flusher.getWaitUntilBufferFlushed()).isEqualTo(60); - assertThat(flusher.getWaitUntilTerminated()).isEqualTo(60); - } + private void assertUnixSocketSender( + UnixSocketSender sender, Path expectedPath, boolean shouldHaveFailureDetector) { + assertThat(sender.getPath()).isEqualTo(expectedPath); + assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(5000); + assertThat(sender.getReadTimeoutMilli()).isEqualTo(5000); - private void assertDefaultRetryableSender(RetryableSender sender, Class expectedBaseClass) - { - assertThat(sender.getRetryStrategy()).isInstanceOf(ExponentialBackOffRetryStrategy.class); - ExponentialBackOffRetryStrategy retryStrategy = (ExponentialBackOffRetryStrategy) sender.getRetryStrategy(); - assertThat(retryStrategy.getMaxRetryCount()).isEqualTo(7); - assertThat(retryStrategy.getBaseIntervalMillis()).isEqualTo(400); - assertThat(sender.getBaseSender()).isInstanceOf(expectedBaseClass); + FailureDetector failureDetector = sender.getFailureDetector(); + if (shouldHaveFailureDetector) { + assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); + assertThat(failureDetector.getFailureDetectStrategy()) + .isInstanceOf(PhiAccrualFailureDetectStrategy.class); + assertThat(failureDetector.getHeartbeater()).isInstanceOf(UnixSocketHeartbeater.class); + { + UnixSocketHeartbeater hb = (UnixSocketHeartbeater) failureDetector.getHeartbeater(); + assertThat(hb.getPath()).isEqualTo(expectedPath); + } + assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); + } else { + assertThat(failureDetector).isNull(); } + } - private void assertUnixSocketSender(UnixSocketSender sender, Path expectedPath, boolean shouldHaveFailureDetector) - { - assertThat(sender.getPath()).isEqualTo(expectedPath); - assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(5000); - assertThat(sender.getReadTimeoutMilli()).isEqualTo(5000); - - FailureDetector failureDetector = sender.getFailureDetector(); - if (shouldHaveFailureDetector) { - assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); - assertThat(failureDetector.getFailureDetectStrategy()).isInstanceOf(PhiAccrualFailureDetectStrategy.class); - assertThat(failureDetector.getHeartbeater()).isInstanceOf(UnixSocketHeartbeater.class); - { - UnixSocketHeartbeater hb = (UnixSocketHeartbeater) failureDetector.getHeartbeater(); - assertThat(hb.getPath()).isEqualTo(expectedPath); - } - assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); - } - else { - assertThat(failureDetector).isNull(); - } - } - - private void assertDefaultFluentdSender(FluentdSender sender, Path expectedPath) - { - assertThat(sender).isInstanceOf(RetryableSender.class); - RetryableSender retryableSender = (RetryableSender) sender; - assertDefaultRetryableSender(retryableSender, UnixSocketSender.class); - assertUnixSocketSender((UnixSocketSender) retryableSender.getBaseSender(), expectedPath, false); - } + private void assertDefaultFluentdSender(FluentdSender sender, Path expectedPath) { + assertThat(sender).isInstanceOf(RetryableSender.class); + RetryableSender retryableSender = (RetryableSender) sender; + assertDefaultRetryableSender(retryableSender, UnixSocketSender.class); + assertUnixSocketSender((UnixSocketSender) retryableSender.getBaseSender(), expectedPath, false); + } - @Test - void build() - throws IOException - { - Path socketPath = Paths.get(System.getProperty("java.io.tmpdir"), "foo/bar.socket"); - try (Fluency fluency = new FluencyExtBuilderForFluentd().build(socketPath)) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender((FluentdSender) fluency.getFlusher().getIngester().getSender(), socketPath); - } + @Test + void build() throws IOException { + Path socketPath = Paths.get(System.getProperty("java.io.tmpdir"), "foo/bar.socket"); + try (Fluency fluency = new FluencyExtBuilderForFluentd().build(socketPath)) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (FluentdSender) fluency.getFlusher().getIngester().getSender(), socketPath); } + } } diff --git a/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/MockUnixSocketServer.java b/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/MockUnixSocketServer.java index 90345c3b..7b35962f 100644 --- a/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/MockUnixSocketServer.java +++ b/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/MockUnixSocketServer.java @@ -16,10 +16,6 @@ package org.komamitsu.fluency.fluentd; -import org.komamitsu.fluency.util.Tuple; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.IOException; import java.net.StandardProtocolFamily; @@ -37,300 +33,296 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import org.komamitsu.fluency.util.Tuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class MockUnixSocketServer - implements Closeable -{ - private static final Logger LOG = LoggerFactory.getLogger(MockUnixSocketServer.class); - private Path socketPath = Paths.get(System.getProperty("java.io.tmpdir"), - String.format("fluency-mock-unixsocket-server-%s", UUID.randomUUID())); - - private final AtomicLong lastEventTimeStampMilli = new AtomicLong(); - private final AtomicInteger threadSeqNum = new AtomicInteger(); - private ExecutorService executorService; - private ServerTask serverTask; - // TODO Make this class immutable - private final List tasks = new ArrayList<>(); - - private final List> events = new CopyOnWriteArrayList<>(); - - public List> getEvents() - { - return events; +public class MockUnixSocketServer implements Closeable { + private static final Logger LOG = LoggerFactory.getLogger(MockUnixSocketServer.class); + private Path socketPath = + Paths.get( + System.getProperty("java.io.tmpdir"), + String.format("fluency-mock-unixsocket-server-%s", UUID.randomUUID())); + + private final AtomicLong lastEventTimeStampMilli = new AtomicLong(); + private final AtomicInteger threadSeqNum = new AtomicInteger(); + private ExecutorService executorService; + private ServerTask serverTask; + // TODO Make this class immutable + private final List tasks = new ArrayList<>(); + + private final List> events = + new CopyOnWriteArrayList<>(); + + public List> getEvents() { + return events; + } + + @Override + public void close() throws IOException { + stop(true); + } + + public enum Type { + CONNECT, + RECEIVE, + CLOSE; + } + + protected EventHandler eventHandler() { + return new EventHandler() { + @Override + public void onConnect(SocketChannel acceptSocketChannel) { + events.add(new Tuple<>(Type.CONNECT, null)); + } + + @Override + public void onReceive(SocketChannel acceptSocketChannel, ByteBuffer byteBuffer) { + events.add(new Tuple<>(Type.RECEIVE, byteBuffer.flip().remaining())); + } + + @Override + public void onClose(SocketChannel acceptSocketChannel) { + events.add(new Tuple<>(Type.CLOSE, null)); + } + }; + } + + public Path getSocketPath() { + return socketPath; + } + + public synchronized void start() throws Exception { + if (executorService == null) { + this.executorService = + Executors.newCachedThreadPool( + r -> + new Thread( + r, String.format("accepted-socket-worker-%d", threadSeqNum.getAndAdd(1)))); } - @Override - public void close() - throws IOException - { - stop(true); + if (serverTask == null) { + UnixDomainSocketAddress socketAddress = UnixDomainSocketAddress.of(socketPath); + serverTask = + new ServerTask( + executorService, lastEventTimeStampMilli, eventHandler(), socketAddress, tasks); + executorService.execute(serverTask); + tasks.add(serverTask); } - - public enum Type - { - CONNECT, RECEIVE, CLOSE; + } + + public void waitUntilEventsStop() throws InterruptedException { + for (int i = 0; i < 20; i++) { + if (lastEventTimeStampMilli.get() + 2000 < System.currentTimeMillis()) { + return; + } + TimeUnit.MILLISECONDS.sleep(500); } + throw new IllegalStateException("Events didn't stop in the expected time"); + } - protected EventHandler eventHandler() - { - return new EventHandler() - { - @Override - public void onConnect(SocketChannel acceptSocketChannel) - { - events.add(new Tuple<>(Type.CONNECT, null)); - } - - @Override - public void onReceive(SocketChannel acceptSocketChannel, ByteBuffer byteBuffer) - { - events.add(new Tuple<>(Type.RECEIVE, byteBuffer.flip().remaining())); - } + public synchronized void stop() throws IOException { + stop(false); + } - @Override - public void onClose(SocketChannel acceptSocketChannel) - { - events.add(new Tuple<>(Type.CLOSE, null)); - } - }; + public synchronized void stop(boolean immediate) throws IOException { + LOG.debug("Stopping the mock server... {}", this); + if (executorService == null) { + return; } - - public Path getSocketPath() - { - return socketPath; + executorService.shutdown(); + try { + if (!executorService.awaitTermination(1000, TimeUnit.MILLISECONDS)) { + LOG.debug("Shutting down the mock server and child tasks... {}", this); + executorService.shutdownNow(); + } + } catch (InterruptedException e) { + LOG.warn("ExecutorService.shutdown() was failed: {}", this, e); + Thread.currentThread().interrupt(); } - public synchronized void start() - throws Exception - { - if (executorService == null) { - this.executorService = Executors.newCachedThreadPool(r -> - new Thread(r, String.format("accepted-socket-worker-%d", threadSeqNum.getAndAdd(1)))); - } - - if (serverTask == null) { - UnixDomainSocketAddress socketAddress = UnixDomainSocketAddress.of(socketPath); - serverTask = new ServerTask(executorService, lastEventTimeStampMilli, eventHandler(), socketAddress, tasks); - executorService.execute(serverTask); - tasks.add(serverTask); + if (immediate) { + LOG.debug("Closing related sockets {}", this); + for (Runnable runnable : tasks) { + if (runnable instanceof ServerTask) { + ((ServerTask) runnable).close(); + } else if (runnable instanceof ServerTask.AcceptTask) { + ((ServerTask.AcceptTask) runnable).close(); } + } } - public void waitUntilEventsStop() - throws InterruptedException - { - for (int i = 0; i < 20; i++) { - if (lastEventTimeStampMilli.get() + 2000 < System.currentTimeMillis()) { - return; - } - TimeUnit.MILLISECONDS.sleep(500); - } - throw new IllegalStateException("Events didn't stop in the expected time"); - } + executorService = null; + serverTask = null; + } - public synchronized void stop() - throws IOException - { - stop(false); - } - - public synchronized void stop(boolean immediate) - throws IOException - { - LOG.debug("Stopping the mock server... {}", this); - if (executorService == null) { - return; - } - executorService.shutdown(); - try { - if (!executorService.awaitTermination(1000, TimeUnit.MILLISECONDS)) { - LOG.debug("Shutting down the mock server and child tasks... {}", this); - executorService.shutdownNow(); - } - } - catch (InterruptedException e) { - LOG.warn("ExecutorService.shutdown() was failed: {}", this, e); - Thread.currentThread().interrupt(); - } + private interface EventHandler { + void onConnect(SocketChannel acceptSocket); - if (immediate) { - LOG.debug("Closing related sockets {}", this); - for (Runnable runnable : tasks) { - if (runnable instanceof ServerTask) { - ((ServerTask) runnable).close(); - } else if (runnable instanceof ServerTask.AcceptTask) { - ((ServerTask.AcceptTask) runnable).close(); - } - } - } + void onReceive(SocketChannel acceptSocket, ByteBuffer byteBuffer); - executorService = null; - serverTask = null; - } + void onClose(SocketChannel acceptSocket); + } - private interface EventHandler - { - void onConnect(SocketChannel acceptSocket); + private static class ServerTask implements Runnable { + private final ServerSocketChannel serverSocketChannel; + private final ExecutorService serverExecutorService; + private final EventHandler eventHandler; + private final AtomicLong lastEventTimeStampMilli; + private final List tasks; + private final UnixDomainSocketAddress socketAddress; - void onReceive(SocketChannel acceptSocket, ByteBuffer byteBuffer); + @Override + public String toString() { + return "ServerTask{" + + "serverSocketChannel=" + + serverSocketChannel + + ", lastEventTimeStampMilli=" + + lastEventTimeStampMilli + + ", socketAddress=" + + socketAddress + + '}'; + } - void onClose(SocketChannel acceptSocket); + private ServerTask( + ExecutorService executorService, + AtomicLong lastEventTimeStampMilli, + EventHandler eventHandler, + UnixDomainSocketAddress socketAddress, + List tasks) + throws IOException { + this.serverExecutorService = executorService; + this.lastEventTimeStampMilli = lastEventTimeStampMilli; + this.eventHandler = eventHandler; + this.socketAddress = socketAddress; + this.serverSocketChannel = + ServerSocketChannel.open(StandardProtocolFamily.UNIX).bind(socketAddress); + this.tasks = tasks; } - private static class ServerTask - implements Runnable - { - private final ServerSocketChannel serverSocketChannel; - private final ExecutorService serverExecutorService; - private final EventHandler eventHandler; - private final AtomicLong lastEventTimeStampMilli; - private final List tasks; - private final UnixDomainSocketAddress socketAddress; - - @Override - public String toString() { - return "ServerTask{" + - "serverSocketChannel=" + serverSocketChannel + - ", lastEventTimeStampMilli=" + lastEventTimeStampMilli + - ", socketAddress=" + socketAddress + - '}'; + @Override + public void run() { + try { + while (!serverExecutorService.isShutdown()) { + try { + LOG.debug("ServerTask: accepting... this={}", this); + SocketChannel acceptSocket = serverSocketChannel.accept(); + LOG.debug("ServerTask: accepted. this={}, remote={}", this, acceptSocket); + AcceptTask acceptTask = + new AcceptTask( + serverExecutorService, lastEventTimeStampMilli, eventHandler, acceptSocket); + serverExecutorService.execute(acceptTask); + tasks.add(acceptTask); + } catch (RejectedExecutionException | ClosedByInterruptException e) { + LOG.debug( + "ServerTask: ServerSocketChannel.accept() failed[{}]: this={}", + e.getMessage(), + this); + } catch (IOException e) { + LOG.warn( + "ServerTask: ServerSocketChannel.accept() failed[{}]: this={}", + e.getMessage(), + this); + } } - - private ServerTask( - ExecutorService executorService, - AtomicLong lastEventTimeStampMilli, - EventHandler eventHandler, - UnixDomainSocketAddress socketAddress, - List tasks - ) - throws IOException - { - this.serverExecutorService = executorService; - this.lastEventTimeStampMilli = lastEventTimeStampMilli; - this.eventHandler = eventHandler; - this.socketAddress = socketAddress; - this.serverSocketChannel = ServerSocketChannel.open(StandardProtocolFamily.UNIX).bind(socketAddress); - this.tasks = tasks; + } finally { + try { + LOG.debug("ServerTask: closing. this={}", this); + close(); + } catch (IOException e) { + LOG.warn("ServerTask: close() failed", e); } + } + LOG.info("ServerTask: Finishing ServerTask...: this={}", this); + } - @Override - public void run() - { - try { - while (!serverExecutorService.isShutdown()) { - try { - LOG.debug("ServerTask: accepting... this={}", this); - SocketChannel acceptSocket = serverSocketChannel.accept(); - LOG.debug("ServerTask: accepted. this={}, remote={}", this, acceptSocket); - AcceptTask acceptTask = new AcceptTask(serverExecutorService, lastEventTimeStampMilli, eventHandler, acceptSocket); - serverExecutorService.execute(acceptTask); - tasks.add(acceptTask); - } catch (RejectedExecutionException | ClosedByInterruptException e) { - LOG.debug("ServerTask: ServerSocketChannel.accept() failed[{}]: this={}", e.getMessage(), this); - } catch (IOException e) { - LOG.warn("ServerTask: ServerSocketChannel.accept() failed[{}]: this={}", e.getMessage(), this); - } - } - } - finally { - try { - LOG.debug("ServerTask: closing. this={}", this); - close(); - } catch (IOException e) { - LOG.warn("ServerTask: close() failed", e); - } - } - LOG.info("ServerTask: Finishing ServerTask...: this={}", this); - } + private void close() throws IOException { + try { + serverSocketChannel.close(); + } finally { + Files.deleteIfExists(socketAddress.getPath()); + } + } - private void close() - throws IOException - { + private static class AcceptTask implements Runnable { + private final SocketChannel acceptSocketChannel; + private final EventHandler eventHandler; + private final ExecutorService serverExecutorService; + private final AtomicLong lastEventTimeStampMilli; + + private AcceptTask( + ExecutorService serverExecutorService, + AtomicLong lastEventTimeStampMilli, + EventHandler eventHandler, + SocketChannel acceptSocketChannel) { + this.serverExecutorService = serverExecutorService; + this.lastEventTimeStampMilli = lastEventTimeStampMilli; + this.eventHandler = eventHandler; + this.acceptSocketChannel = acceptSocketChannel; + } + + @Override + public String toString() { + return "AcceptTask{" + + "acceptSocketChannel=" + + acceptSocketChannel + + ", lastEventTimeStampMilli=" + + lastEventTimeStampMilli + + '}'; + } + + private void close() throws IOException { + acceptSocketChannel.close(); + } + + @Override + public void run() { + LOG.debug("AcceptTask: connected. this={}", this); + try { + eventHandler.onConnect(acceptSocketChannel); + ByteBuffer byteBuffer = ByteBuffer.allocate(512 * 1024); + while (!serverExecutorService.isShutdown()) { try { - serverSocketChannel.close(); - } - finally { - Files.deleteIfExists(socketAddress.getPath()); - } - } - - private static class AcceptTask - implements Runnable - { - private final SocketChannel acceptSocketChannel; - private final EventHandler eventHandler; - private final ExecutorService serverExecutorService; - private final AtomicLong lastEventTimeStampMilli; - - private AcceptTask(ExecutorService serverExecutorService, AtomicLong lastEventTimeStampMilli, EventHandler eventHandler, SocketChannel acceptSocketChannel) - { - this.serverExecutorService = serverExecutorService; - this.lastEventTimeStampMilli = lastEventTimeStampMilli; - this.eventHandler = eventHandler; - this.acceptSocketChannel = acceptSocketChannel; - } - - @Override - public String toString() - { - return "AcceptTask{" + - "acceptSocketChannel=" + acceptSocketChannel + - ", lastEventTimeStampMilli=" + lastEventTimeStampMilli + - '}'; - } - - private void close() - throws IOException - { - acceptSocketChannel.close(); - } - - @Override - public void run() - { - LOG.debug("AcceptTask: connected. this={}", this); + int len = acceptSocketChannel.read(byteBuffer); + if (len <= 0) { + LOG.debug( + "AcceptTask: closed. len={}, this={}, remote={}", + len, + this, + acceptSocketChannel); + eventHandler.onClose(acceptSocketChannel); try { - eventHandler.onConnect(acceptSocketChannel); - ByteBuffer byteBuffer = ByteBuffer.allocate(512 * 1024); - while (!serverExecutorService.isShutdown()) { - try { - int len = acceptSocketChannel.read(byteBuffer); - if (len <= 0) { - LOG.debug("AcceptTask: closed. len={}, this={}, remote={}", len, this, acceptSocketChannel); - eventHandler.onClose(acceptSocketChannel); - try { - close(); - } - catch (IOException e) { - LOG.warn("AcceptTask: close() failed: this={}", this, e); - } - break; - } - else { - eventHandler.onReceive(acceptSocketChannel, byteBuffer); - lastEventTimeStampMilli.set(System.currentTimeMillis()); - } - } - catch (IOException e) { - LOG.warn("AcceptTask: recv() failed: this={}, message={}, cause={}", - this, e.getMessage(), e.getCause() == null ? "" : e.getCause().getMessage()); - if (!acceptSocketChannel.isOpen()) { - eventHandler.onClose(acceptSocketChannel); - throw new RuntimeException(e); - } - } - } - } - finally { - try { - LOG.debug("AcceptTask: Finished. Closing... this={}, remote={}", this, acceptSocketChannel); - close(); - } - catch (IOException e) { - LOG.warn("AcceptTask: close() failed", e); - } + close(); + } catch (IOException e) { + LOG.warn("AcceptTask: close() failed: this={}", this, e); } + break; + } else { + eventHandler.onReceive(acceptSocketChannel, byteBuffer); + lastEventTimeStampMilli.set(System.currentTimeMillis()); + } + } catch (IOException e) { + LOG.warn( + "AcceptTask: recv() failed: this={}, message={}, cause={}", + this, + e.getMessage(), + e.getCause() == null ? "" : e.getCause().getMessage()); + if (!acceptSocketChannel.isOpen()) { + eventHandler.onClose(acceptSocketChannel); + throw new RuntimeException(e); + } } + } + } finally { + try { + LOG.debug( + "AcceptTask: Finished. Closing... this={}, remote={}", this, acceptSocketChannel); + close(); + } catch (IOException e) { + LOG.warn("AcceptTask: close() failed", e); + } } + } } + } } diff --git a/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/UnixSocketSenderTest.java b/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/UnixSocketSenderTest.java index 4c5fa74c..1c61e62d 100644 --- a/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/UnixSocketSenderTest.java +++ b/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/UnixSocketSenderTest.java @@ -16,14 +16,8 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.fluentd.MockUnixSocketServer; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; -import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.UnixSocketHeartbeater; -import org.komamitsu.fluency.util.Tuple; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; import java.io.IOException; import java.net.SocketException; @@ -35,259 +29,259 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.fluentd.MockUnixSocketServer; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; +import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.UnixSocketHeartbeater; +import org.komamitsu.fluency.util.Tuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.*; - -class UnixSocketSenderTest -{ - private static final Logger LOG = LoggerFactory.getLogger(UnixSocketSenderTest.class); - - - @Test - void testSend() - throws Exception - { - testSendBase(socketPath -> { - UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); - senderConfig.setPath(socketPath); - return new UnixSocketSender(senderConfig); - }, - count -> assertThat(count).isEqualTo(1), - count -> assertThat(count).isEqualTo(1)); - } - - @Test - void testSendWithHeartbeart() - throws Exception - { - testSendBase(socketPath -> { - UnixSocketHeartbeater.Config hbConfig = new UnixSocketHeartbeater.Config(); - hbConfig.setPath(socketPath); - hbConfig.setIntervalMillis(400); - - UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); - senderConfig.setPath(socketPath); - - return new UnixSocketSender(senderConfig, - new FailureDetector( - new PhiAccrualFailureDetectStrategy(), - new UnixSocketHeartbeater(hbConfig))); - }, - count -> assertThat(count).isGreaterThan(1), - count -> assertThat(count).isGreaterThan(1)); - } - - private void testSendBase( - SenderCreator senderCreator, - Consumer connectCountAssertion, - Consumer closeCountAssertion) - throws Exception - { - try (MockUnixSocketServer server = new MockUnixSocketServer()) { - server.start(); - - int concurrency = 10; - final int reqNum = 5000; - final CountDownLatch latch = new CountDownLatch(concurrency); - UnixSocketSender sender = senderCreator.create(server.getSocketPath()); - - // To receive heartbeat at least once - TimeUnit.MILLISECONDS.sleep(500); - - final ExecutorService senderExecutorService = Executors.newCachedThreadPool(); - for (int i = 0; i < concurrency; i++) { - senderExecutorService.execute(() -> { - try { - byte[] bytes = "0123456789".getBytes(); - - for (int j = 0; j < reqNum; j++) { - sender.send(ByteBuffer.wrap(bytes)); - } - latch.countDown(); - } catch (IOException e) { - LOG.error("Failed to send data", e); - } - }); - } - - assertTrue(latch.await(4, TimeUnit.SECONDS)); - sender.close(); - - server.waitUntilEventsStop(); - server.stop(); - - int connectCount = 0; - int closeCount = 0; - long recvCount = 0; - long recvLen = 0; - for (Tuple event : server.getEvents()) { - switch (event.getFirst()) { - case CONNECT: - connectCount++; - break; - case CLOSE: - closeCount++; - break; - case RECEIVE: - recvCount++; - recvLen += event.getSecond(); - break; +class UnixSocketSenderTest { + private static final Logger LOG = LoggerFactory.getLogger(UnixSocketSenderTest.class); + + @Test + void testSend() throws Exception { + testSendBase( + socketPath -> { + UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); + senderConfig.setPath(socketPath); + return new UnixSocketSender(senderConfig); + }, + count -> assertThat(count).isEqualTo(1), + count -> assertThat(count).isEqualTo(1)); + } + + @Test + void testSendWithHeartbeart() throws Exception { + testSendBase( + socketPath -> { + UnixSocketHeartbeater.Config hbConfig = new UnixSocketHeartbeater.Config(); + hbConfig.setPath(socketPath); + hbConfig.setIntervalMillis(400); + + UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); + senderConfig.setPath(socketPath); + + return new UnixSocketSender( + senderConfig, + new FailureDetector( + new PhiAccrualFailureDetectStrategy(), new UnixSocketHeartbeater(hbConfig))); + }, + count -> assertThat(count).isGreaterThan(1), + count -> assertThat(count).isGreaterThan(1)); + } + + private void testSendBase( + SenderCreator senderCreator, + Consumer connectCountAssertion, + Consumer closeCountAssertion) + throws Exception { + try (MockUnixSocketServer server = new MockUnixSocketServer()) { + server.start(); + + int concurrency = 10; + final int reqNum = 5000; + final CountDownLatch latch = new CountDownLatch(concurrency); + UnixSocketSender sender = senderCreator.create(server.getSocketPath()); + + // To receive heartbeat at least once + TimeUnit.MILLISECONDS.sleep(500); + + final ExecutorService senderExecutorService = Executors.newCachedThreadPool(); + for (int i = 0; i < concurrency; i++) { + senderExecutorService.execute( + () -> { + try { + byte[] bytes = "0123456789".getBytes(); + + for (int j = 0; j < reqNum; j++) { + sender.send(ByteBuffer.wrap(bytes)); } - } - LOG.debug("recvCount={}", recvCount); - - connectCountAssertion.accept(connectCount); - assertThat(recvLen).isEqualTo((long) concurrency * reqNum * 10); - closeCountAssertion.accept(closeCount); + latch.countDown(); + } catch (IOException e) { + LOG.error("Failed to send data", e); + } + }); + } + + assertTrue(latch.await(4, TimeUnit.SECONDS)); + sender.close(); + + server.waitUntilEventsStop(); + server.stop(); + + int connectCount = 0; + int closeCount = 0; + long recvCount = 0; + long recvLen = 0; + for (Tuple event : server.getEvents()) { + switch (event.getFirst()) { + case CONNECT: + connectCount++; + break; + case CLOSE: + closeCount++; + break; + case RECEIVE: + recvCount++; + recvLen += event.getSecond(); + break; } - } + } + LOG.debug("recvCount={}", recvCount); - @Test - void testReadTimeout() - throws Exception - { - try (MockUnixSocketServer server = new MockUnixSocketServer()) { - server.start(); - - try { - final CountDownLatch latch = new CountDownLatch(1); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.execute(() -> { - UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); - senderConfig.setPath(server.getSocketPath()); - senderConfig.setReadTimeoutMilli(1000); - UnixSocketSender sender = new UnixSocketSender(senderConfig); - try { - sender.sendWithAck(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), "Waiting ack forever"); - } catch (Throwable e) { - if (e instanceof SocketTimeoutException) { - latch.countDown(); - } else { - throw new RuntimeException(e); - } - } - }); - assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); - } finally { - server.stop(); - } - } + connectCountAssertion.accept(connectCount); + assertThat(recvLen).isEqualTo((long) concurrency * reqNum * 10); + closeCountAssertion.accept(closeCount); } - - private Throwable extractRootCause(Throwable exception) - { - Throwable e = exception; - while (e.getCause() != null) { - e = e.getCause(); - } - return e; + } + + @Test + void testReadTimeout() throws Exception { + try (MockUnixSocketServer server = new MockUnixSocketServer()) { + server.start(); + + try { + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.execute( + () -> { + UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); + senderConfig.setPath(server.getSocketPath()); + senderConfig.setReadTimeoutMilli(1000); + UnixSocketSender sender = new UnixSocketSender(senderConfig); + try { + sender.sendWithAck( + Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), + "Waiting ack forever"); + } catch (Throwable e) { + if (e instanceof SocketTimeoutException) { + latch.countDown(); + } else { + throw new RuntimeException(e); + } + } + }); + assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); + } finally { + server.stop(); + } } + } - @Test - void testDisconnBeforeRecv() - throws Exception - { - try (MockUnixSocketServer server = new MockUnixSocketServer()) { - server.start(); - - try { - final CountDownLatch latch = new CountDownLatch(1); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.execute(() -> { - UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); - senderConfig.setPath(server.getSocketPath()); - senderConfig.setReadTimeoutMilli(4000); - UnixSocketSender sender = new UnixSocketSender(senderConfig); - try { - sender.sendWithAck(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), "Waiting ack forever"); - } catch (Throwable e) { - Throwable rootCause = extractRootCause(e); - if (rootCause instanceof SocketException && rootCause.getMessage().toLowerCase().contains("disconnected")) { - latch.countDown(); - } else { - throw new RuntimeException(e); - } - } - }); + private Throwable extractRootCause(Throwable exception) { + Throwable e = exception; + while (e.getCause() != null) { + e = e.getCause(); + } + return e; + } + + @Test + void testDisconnBeforeRecv() throws Exception { + try (MockUnixSocketServer server = new MockUnixSocketServer()) { + server.start(); + + try { + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.execute( + () -> { + UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); + senderConfig.setPath(server.getSocketPath()); + senderConfig.setReadTimeoutMilli(4000); + UnixSocketSender sender = new UnixSocketSender(senderConfig); + try { + sender.sendWithAck( + Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), + "Waiting ack forever"); + } catch (Throwable e) { + Throwable rootCause = extractRootCause(e); + if (rootCause instanceof SocketException + && rootCause.getMessage().toLowerCase().contains("disconnected")) { + latch.countDown(); + } else { + throw new RuntimeException(e); + } + } + }); - TimeUnit.MILLISECONDS.sleep(1000); - server.stop(true); + TimeUnit.MILLISECONDS.sleep(1000); + server.stop(true); - assertTrue(latch.await(8000, TimeUnit.MILLISECONDS)); - } finally { - server.stop(); - } - } + assertTrue(latch.await(8000, TimeUnit.MILLISECONDS)); + } finally { + server.stop(); + } } - - @Test - void testClose() - throws Exception - { - try (MockUnixSocketServer server = new MockUnixSocketServer()) { - server.start(); - - try { - final AtomicLong duration = new AtomicLong(); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - Future future = executorService.submit(() -> { - UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); - senderConfig.setPath(server.getSocketPath()); - senderConfig.setWaitBeforeCloseMilli(1500); - UnixSocketSender sender = new UnixSocketSender(senderConfig); - long start; - try { - sender.send(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes("UTF-8")))); - start = System.currentTimeMillis(); - sender.close(); - duration.set(System.currentTimeMillis() - start); - } catch (Exception e) { - LOG.error("Unexpected exception", e); - } - - return null; + } + + @Test + void testClose() throws Exception { + try (MockUnixSocketServer server = new MockUnixSocketServer()) { + server.start(); + + try { + final AtomicLong duration = new AtomicLong(); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + Future future = + executorService.submit( + () -> { + UnixSocketSender.Config senderConfig = new UnixSocketSender.Config(); + senderConfig.setPath(server.getSocketPath()); + senderConfig.setWaitBeforeCloseMilli(1500); + UnixSocketSender sender = new UnixSocketSender(senderConfig); + long start; + try { + sender.send(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes("UTF-8")))); + start = System.currentTimeMillis(); + sender.close(); + duration.set(System.currentTimeMillis() - start); + } catch (Exception e) { + LOG.error("Unexpected exception", e); + } + + return null; }); - future.get(3000, TimeUnit.MILLISECONDS); - assertTrue(duration.get() > 1000 && duration.get() < 2000); - } finally { - server.stop(); - } - } + future.get(3000, TimeUnit.MILLISECONDS); + assertTrue(duration.get() > 1000 && duration.get() < 2000); + } finally { + server.stop(); + } } + } + + @Test + void testConfig() { + UnixSocketSender.Config config = new UnixSocketSender.Config(); + assertEquals(1000, config.getWaitBeforeCloseMilli()); + // TODO: Add others later + } - @Test - void testConfig() + @Test + void validateConfig() { { - UnixSocketSender.Config config = new UnixSocketSender.Config(); - assertEquals(1000, config.getWaitBeforeCloseMilli()); - // TODO: Add others later + UnixSocketSender.Config config = new UnixSocketSender.Config(); + config.setConnectionTimeoutMilli(9); + assertThrows(IllegalArgumentException.class, () -> new UnixSocketSender(config)); } - @Test - void validateConfig() { - { - UnixSocketSender.Config config = new UnixSocketSender.Config(); - config.setConnectionTimeoutMilli(9); - assertThrows(IllegalArgumentException.class, () -> new UnixSocketSender(config)); - } - - { - UnixSocketSender.Config config = new UnixSocketSender.Config(); - config.setReadTimeoutMilli(9); - assertThrows(IllegalArgumentException.class, () -> new UnixSocketSender(config)); - } - - { - UnixSocketSender.Config config = new UnixSocketSender.Config(); - config.setWaitBeforeCloseMilli(-1); - assertThrows(IllegalArgumentException.class, () -> new UnixSocketSender(config)); - } + UnixSocketSender.Config config = new UnixSocketSender.Config(); + config.setReadTimeoutMilli(9); + assertThrows(IllegalArgumentException.class, () -> new UnixSocketSender(config)); } - interface SenderCreator { - UnixSocketSender create(Path socketPath); + UnixSocketSender.Config config = new UnixSocketSender.Config(); + config.setWaitBeforeCloseMilli(-1); + assertThrows(IllegalArgumentException.class, () -> new UnixSocketSender(config)); } + } + + interface SenderCreator { + UnixSocketSender create(Path socketPath); + } } diff --git a/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UnixSocketHeartbeaterTest.java b/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UnixSocketHeartbeaterTest.java index 3eb95a52..c10a529d 100644 --- a/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UnixSocketHeartbeaterTest.java +++ b/fluency-fluentd-ext/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UnixSocketHeartbeaterTest.java @@ -16,9 +16,8 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.net.StandardProtocolFamily; @@ -32,108 +31,97 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - +class UnixSocketHeartbeaterTest { + private Path socketPath; + private ServerSocketChannel serverSocketChannel; -class UnixSocketHeartbeaterTest -{ - private Path socketPath; - private ServerSocketChannel serverSocketChannel; + @BeforeEach + void setUp() throws IOException { + socketPath = + Paths.get( + System.getProperty("java.io.tmpdir"), + String.format("fluency-unixsocket-hb-test-%s", UUID.randomUUID())); + UnixDomainSocketAddress socketAddress = UnixDomainSocketAddress.of(socketPath); - @BeforeEach - void setUp() - throws IOException - { - socketPath = Paths.get(System.getProperty("java.io.tmpdir"), - String.format("fluency-unixsocket-hb-test-%s", UUID.randomUUID())); - UnixDomainSocketAddress socketAddress = UnixDomainSocketAddress.of(socketPath); + serverSocketChannel = ServerSocketChannel.open(StandardProtocolFamily.UNIX); + serverSocketChannel.bind(socketAddress); + } - serverSocketChannel = ServerSocketChannel.open(StandardProtocolFamily.UNIX); - serverSocketChannel.bind(socketAddress); - } + @AfterEach + void tearDown() throws IOException { + Files.deleteIfExists(socketPath); + } - @AfterEach - void tearDown() - throws IOException - { - Files.deleteIfExists(socketPath); - } - - @Test - void testHeartbeaterUp() - throws IOException, InterruptedException - { - final CountDownLatch latch = new CountDownLatch(2); - Executors.newSingleThreadExecutor().execute(() -> { - try { + @Test + void testHeartbeaterUp() throws IOException, InterruptedException { + final CountDownLatch latch = new CountDownLatch(2); + Executors.newSingleThreadExecutor() + .execute( + () -> { + try { serverSocketChannel.accept(); latch.countDown(); - } - catch (IOException e) { + } catch (IOException e) { e.printStackTrace(); - } - }); + } + }); - UnixSocketHeartbeater.Config config = new UnixSocketHeartbeater.Config(); - config.setPath(socketPath); - config.setIntervalMillis(500); - try (UnixSocketHeartbeater heartbeater = new UnixSocketHeartbeater(config)) { - final AtomicInteger pongCounter = new AtomicInteger(); - final AtomicInteger failureCounter = new AtomicInteger(); - heartbeater.setCallback(new Heartbeater.Callback() - { - @Override - public void onHeartbeat() - { - pongCounter.incrementAndGet(); - latch.countDown(); - } + UnixSocketHeartbeater.Config config = new UnixSocketHeartbeater.Config(); + config.setPath(socketPath); + config.setIntervalMillis(500); + try (UnixSocketHeartbeater heartbeater = new UnixSocketHeartbeater(config)) { + final AtomicInteger pongCounter = new AtomicInteger(); + final AtomicInteger failureCounter = new AtomicInteger(); + heartbeater.setCallback( + new Heartbeater.Callback() { + @Override + public void onHeartbeat() { + pongCounter.incrementAndGet(); + latch.countDown(); + } - @Override - public void onFailure(Throwable cause) - { - failureCounter.incrementAndGet(); - } - }); - heartbeater.start(); - assertTrue(latch.await(5, TimeUnit.SECONDS)); - assertTrue(0 < pongCounter.get() && pongCounter.get() < 3); - assertEquals(0, failureCounter.get()); - } + @Override + public void onFailure(Throwable cause) { + failureCounter.incrementAndGet(); + } + }); + heartbeater.start(); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(0 < pongCounter.get() && pongCounter.get() < 3); + assertEquals(0, failureCounter.get()); } + } - @Test - void testHeartbeaterDown() - throws IOException, InterruptedException - { - serverSocketChannel.close(); + @Test + void testHeartbeaterDown() throws IOException, InterruptedException { + serverSocketChannel.close(); - UnixSocketHeartbeater.Config config = new UnixSocketHeartbeater.Config(); - config.setPath(socketPath); - config.setIntervalMillis(500); - try (UnixSocketHeartbeater heartbeater = new UnixSocketHeartbeater(config)) { - final AtomicInteger pongCounter = new AtomicInteger(); - final AtomicInteger failureCounter = new AtomicInteger(); - heartbeater.setCallback(new Heartbeater.Callback() - { - @Override - public void onHeartbeat() - { - pongCounter.incrementAndGet(); - } + UnixSocketHeartbeater.Config config = new UnixSocketHeartbeater.Config(); + config.setPath(socketPath); + config.setIntervalMillis(500); + try (UnixSocketHeartbeater heartbeater = new UnixSocketHeartbeater(config)) { + final AtomicInteger pongCounter = new AtomicInteger(); + final AtomicInteger failureCounter = new AtomicInteger(); + heartbeater.setCallback( + new Heartbeater.Callback() { + @Override + public void onHeartbeat() { + pongCounter.incrementAndGet(); + } - @Override - public void onFailure(Throwable cause) - { - failureCounter.incrementAndGet(); - } - }); - heartbeater.start(); - TimeUnit.SECONDS.sleep(1); - assertEquals(0, pongCounter.get()); - assertTrue(failureCounter.get() > 0); - } + @Override + public void onFailure(Throwable cause) { + failureCounter.incrementAndGet(); + } + }); + heartbeater.start(); + TimeUnit.SECONDS.sleep(1); + assertEquals(0, pongCounter.get()); + assertTrue(failureCounter.get() > 0); } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/FluencyBuilderForFluentd.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/FluencyBuilderForFluentd.java index eac64f4d..b746701d 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/FluencyBuilderForFluentd.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/FluencyBuilderForFluentd.java @@ -16,6 +16,10 @@ package org.komamitsu.fluency.fluentd; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; +import javax.net.ssl.SSLSocketFactory; import org.komamitsu.fluency.Fluency; import org.komamitsu.fluency.fluentd.ingester.FluentdIngester; import org.komamitsu.fluency.fluentd.ingester.sender.FluentdSender; @@ -31,249 +35,213 @@ import org.komamitsu.fluency.fluentd.recordformat.FluentdRecordFormatter; import org.komamitsu.fluency.ingester.Ingester; -import java.net.InetSocketAddress; -import javax.net.ssl.SSLSocketFactory; -import java.util.ArrayList; -import java.util.List; - -public class FluencyBuilderForFluentd - extends org.komamitsu.fluency.FluencyBuilder -{ - private Integer senderMaxRetryCount; - private Integer senderBaseRetryIntervalMillis; - private Integer senderMaxRetryIntervalMillis; - private boolean ackResponseMode; - private boolean sslEnabled; - private SSLSocketFactory sslSocketFactory; - protected Integer connectionTimeoutMilli; - protected Integer readTimeoutMilli; - protected FluentdRecordFormatter recordFormatter = new FluentdRecordFormatter(); - - public Integer getSenderMaxRetryCount() - { - return senderMaxRetryCount; - } - - public Integer getSenderBaseRetryIntervalMillis() - { - return senderBaseRetryIntervalMillis; - } - - public Integer getSenderMaxRetryIntervalMillis() - { - return senderMaxRetryIntervalMillis; - } - - public void setSenderBaseRetryIntervalMillis(Integer senderBaseRetryIntervalMillis) - { - this.senderBaseRetryIntervalMillis = senderBaseRetryIntervalMillis; - } - - public void setSenderMaxRetryIntervalMillis(Integer senderMaxRetryIntervalMillis) - { - this.senderMaxRetryIntervalMillis = senderMaxRetryIntervalMillis; - } - - public void setSenderMaxRetryCount(Integer senderMaxRetryCount) - { - this.senderMaxRetryCount = senderMaxRetryCount; - } - - public boolean isAckResponseMode() - { - return ackResponseMode; - } - - public void setAckResponseMode(boolean ackResponseMode) - { - this.ackResponseMode = ackResponseMode; - } - - public boolean isSslEnabled() - { - return sslEnabled; - } - - public void setSslEnabled(boolean sslEnabled) - { - this.sslEnabled = sslEnabled; - } - - public void setSslSocketFactory(SSLSocketFactory sslSocketFactory) - { - this.sslSocketFactory = sslSocketFactory; - } - - public Integer getConnectionTimeoutMilli() - { - return connectionTimeoutMilli; - } - - public void setConnectionTimeoutMilli(Integer connectionTimeoutMilli) - { - this.connectionTimeoutMilli = connectionTimeoutMilli; - } - - public Integer getReadTimeoutMilli() - { - return readTimeoutMilli; - } - - public void setReadTimeoutMilli(Integer readTimeoutMilli) - { - this.readTimeoutMilli = readTimeoutMilli; - } - - public FluentdRecordFormatter getRecordFormatter() - { - return recordFormatter; - } - - public void setRecordFormatter(FluentdRecordFormatter recordFormatter) - { - this.recordFormatter = recordFormatter; - } - - public Fluency build(String host, int port) - { - return buildFromIngester( - recordFormatter, - buildIngester(createBaseSender(host, port))); - } - - public Fluency build(int port) - { - return buildFromIngester( - recordFormatter, - buildIngester(createBaseSender(null, port))); - } - - public Fluency build() - { - return buildFromIngester( - recordFormatter, - buildIngester(createBaseSender(null, null))); - } - - public Fluency build(List servers) - { - List senders = new ArrayList<>(); - for (InetSocketAddress server : servers) { - senders.add(createBaseSender(server.getHostName(), server.getPort(), true)); - } - return buildFromIngester( - recordFormatter, - buildIngester(new MultiSender(senders))); - } - - private FluentdSender createBaseSender(String host, Integer port) - { - return createBaseSender(host, port, false); - } - - private FluentdSender createBaseSender(String host, Integer port, boolean withHeartBeater) - { - if (withHeartBeater && port == null) { - throw new IllegalArgumentException("`port` should be specified when using heartbeat"); - } - - if (isSslEnabled()) { - SSLSender.Config senderConfig = new SSLSender.Config(); - FailureDetector failureDetector = null; - if (host != null) { - senderConfig.setHost(host); - } - if (port != null) { - senderConfig.setPort(port); - } - if (sslSocketFactory != null) { - senderConfig.setSslSocketFactory(sslSocketFactory); - } - if (withHeartBeater) { - SSLHeartbeater.Config hbConfig = new SSLHeartbeater.Config(); - hbConfig.setHost(host); - hbConfig.setPort(port); - - if (sslSocketFactory != null) { - hbConfig.setSslSocketFactory(sslSocketFactory); - } - - SSLHeartbeater heartbeater = new SSLHeartbeater(hbConfig); - failureDetector = new FailureDetector(new PhiAccrualFailureDetectStrategy(), heartbeater); - } - if (connectionTimeoutMilli != null) { - senderConfig.setConnectionTimeoutMilli(connectionTimeoutMilli); - } - if (readTimeoutMilli != null) { - senderConfig.setReadTimeoutMilli(readTimeoutMilli); - } - return new SSLSender(senderConfig, failureDetector); - } - else { - TCPSender.Config senderConfig = new TCPSender.Config(); - FailureDetector failureDetector = null; - if (host != null) { - senderConfig.setHost(host); - } - if (port != null) { - senderConfig.setPort(port); - } - if (withHeartBeater) { - TCPHeartbeater.Config hbConfig = new TCPHeartbeater.Config(); - hbConfig.setHost(host); - hbConfig.setPort(port); - TCPHeartbeater heartbeater = new TCPHeartbeater(hbConfig); - failureDetector = new FailureDetector(new PhiAccrualFailureDetectStrategy(), heartbeater); - } - if (connectionTimeoutMilli != null) { - senderConfig.setConnectionTimeoutMilli(connectionTimeoutMilli); - } - if (readTimeoutMilli != null) { - senderConfig.setReadTimeoutMilli(readTimeoutMilli); - } - return new TCPSender(senderConfig, failureDetector); - } - } - - @Override - public String toString() - { - return "FluencyBuilder{" + - "senderMaxRetryCount=" + senderMaxRetryCount + - ", ackResponseMode=" + ackResponseMode + - ", sslEnabled=" + sslEnabled + - "} " + super.toString(); - } - - protected Ingester buildIngester(FluentdSender baseSender) - { - ExponentialBackOffRetryStrategy.Config retryStrategyConfig = - new ExponentialBackOffRetryStrategy.Config(); - - if (getSenderMaxRetryCount() != null) { - retryStrategyConfig.setMaxRetryCount(getSenderMaxRetryCount()); - } - - if (getSenderBaseRetryIntervalMillis() != null) { - retryStrategyConfig.setBaseIntervalMillis(getSenderBaseRetryIntervalMillis()); - } - - if (getSenderMaxRetryIntervalMillis() != null) { - retryStrategyConfig.setMaxIntervalMillis(getSenderMaxRetryIntervalMillis()); +public class FluencyBuilderForFluentd extends org.komamitsu.fluency.FluencyBuilder { + private Integer senderMaxRetryCount; + private Integer senderBaseRetryIntervalMillis; + private Integer senderMaxRetryIntervalMillis; + private boolean ackResponseMode; + private boolean sslEnabled; + private SSLSocketFactory sslSocketFactory; + protected Integer connectionTimeoutMilli; + protected Integer readTimeoutMilli; + protected FluentdRecordFormatter recordFormatter = new FluentdRecordFormatter(); + + public Integer getSenderMaxRetryCount() { + return senderMaxRetryCount; + } + + public Integer getSenderBaseRetryIntervalMillis() { + return senderBaseRetryIntervalMillis; + } + + public Integer getSenderMaxRetryIntervalMillis() { + return senderMaxRetryIntervalMillis; + } + + public void setSenderBaseRetryIntervalMillis(Integer senderBaseRetryIntervalMillis) { + this.senderBaseRetryIntervalMillis = senderBaseRetryIntervalMillis; + } + + public void setSenderMaxRetryIntervalMillis(Integer senderMaxRetryIntervalMillis) { + this.senderMaxRetryIntervalMillis = senderMaxRetryIntervalMillis; + } + + public void setSenderMaxRetryCount(Integer senderMaxRetryCount) { + this.senderMaxRetryCount = senderMaxRetryCount; + } + + public boolean isAckResponseMode() { + return ackResponseMode; + } + + public void setAckResponseMode(boolean ackResponseMode) { + this.ackResponseMode = ackResponseMode; + } + + public boolean isSslEnabled() { + return sslEnabled; + } + + public void setSslEnabled(boolean sslEnabled) { + this.sslEnabled = sslEnabled; + } + + public void setSslSocketFactory(SSLSocketFactory sslSocketFactory) { + this.sslSocketFactory = sslSocketFactory; + } + + public Integer getConnectionTimeoutMilli() { + return connectionTimeoutMilli; + } + + public void setConnectionTimeoutMilli(Integer connectionTimeoutMilli) { + this.connectionTimeoutMilli = connectionTimeoutMilli; + } + + public Integer getReadTimeoutMilli() { + return readTimeoutMilli; + } + + public void setReadTimeoutMilli(Integer readTimeoutMilli) { + this.readTimeoutMilli = readTimeoutMilli; + } + + public FluentdRecordFormatter getRecordFormatter() { + return recordFormatter; + } + + public void setRecordFormatter(FluentdRecordFormatter recordFormatter) { + this.recordFormatter = recordFormatter; + } + + public Fluency build(String host, int port) { + return buildFromIngester(recordFormatter, buildIngester(createBaseSender(host, port))); + } + + public Fluency build(int port) { + return buildFromIngester(recordFormatter, buildIngester(createBaseSender(null, port))); + } + + public Fluency build() { + return buildFromIngester(recordFormatter, buildIngester(createBaseSender(null, null))); + } + + public Fluency build(List servers) { + List senders = new ArrayList<>(); + for (InetSocketAddress server : servers) { + senders.add(createBaseSender(server.getHostName(), server.getPort(), true)); + } + return buildFromIngester(recordFormatter, buildIngester(new MultiSender(senders))); + } + + private FluentdSender createBaseSender(String host, Integer port) { + return createBaseSender(host, port, false); + } + + private FluentdSender createBaseSender(String host, Integer port, boolean withHeartBeater) { + if (withHeartBeater && port == null) { + throw new IllegalArgumentException("`port` should be specified when using heartbeat"); + } + + if (isSslEnabled()) { + SSLSender.Config senderConfig = new SSLSender.Config(); + FailureDetector failureDetector = null; + if (host != null) { + senderConfig.setHost(host); + } + if (port != null) { + senderConfig.setPort(port); + } + if (sslSocketFactory != null) { + senderConfig.setSslSocketFactory(sslSocketFactory); + } + if (withHeartBeater) { + SSLHeartbeater.Config hbConfig = new SSLHeartbeater.Config(); + hbConfig.setHost(host); + hbConfig.setPort(port); + + if (sslSocketFactory != null) { + hbConfig.setSslSocketFactory(sslSocketFactory); } - RetryableSender.Config senderConfig = new RetryableSender.Config(); - - if (getErrorHandler() != null) { - senderConfig.setErrorHandler(getErrorHandler()); - } - - RetryableSender retryableSender = new RetryableSender(senderConfig, baseSender, - new ExponentialBackOffRetryStrategy(retryStrategyConfig)); - - FluentdIngester.Config ingesterConfig = new FluentdIngester.Config(); - ingesterConfig.setAckResponseMode(isAckResponseMode()); - - return new FluentdIngester(ingesterConfig, retryableSender); - } + SSLHeartbeater heartbeater = new SSLHeartbeater(hbConfig); + failureDetector = new FailureDetector(new PhiAccrualFailureDetectStrategy(), heartbeater); + } + if (connectionTimeoutMilli != null) { + senderConfig.setConnectionTimeoutMilli(connectionTimeoutMilli); + } + if (readTimeoutMilli != null) { + senderConfig.setReadTimeoutMilli(readTimeoutMilli); + } + return new SSLSender(senderConfig, failureDetector); + } else { + TCPSender.Config senderConfig = new TCPSender.Config(); + FailureDetector failureDetector = null; + if (host != null) { + senderConfig.setHost(host); + } + if (port != null) { + senderConfig.setPort(port); + } + if (withHeartBeater) { + TCPHeartbeater.Config hbConfig = new TCPHeartbeater.Config(); + hbConfig.setHost(host); + hbConfig.setPort(port); + TCPHeartbeater heartbeater = new TCPHeartbeater(hbConfig); + failureDetector = new FailureDetector(new PhiAccrualFailureDetectStrategy(), heartbeater); + } + if (connectionTimeoutMilli != null) { + senderConfig.setConnectionTimeoutMilli(connectionTimeoutMilli); + } + if (readTimeoutMilli != null) { + senderConfig.setReadTimeoutMilli(readTimeoutMilli); + } + return new TCPSender(senderConfig, failureDetector); + } + } + + @Override + public String toString() { + return "FluencyBuilder{" + + "senderMaxRetryCount=" + + senderMaxRetryCount + + ", ackResponseMode=" + + ackResponseMode + + ", sslEnabled=" + + sslEnabled + + "} " + + super.toString(); + } + + protected Ingester buildIngester(FluentdSender baseSender) { + ExponentialBackOffRetryStrategy.Config retryStrategyConfig = + new ExponentialBackOffRetryStrategy.Config(); + + if (getSenderMaxRetryCount() != null) { + retryStrategyConfig.setMaxRetryCount(getSenderMaxRetryCount()); + } + + if (getSenderBaseRetryIntervalMillis() != null) { + retryStrategyConfig.setBaseIntervalMillis(getSenderBaseRetryIntervalMillis()); + } + + if (getSenderMaxRetryIntervalMillis() != null) { + retryStrategyConfig.setMaxIntervalMillis(getSenderMaxRetryIntervalMillis()); + } + + RetryableSender.Config senderConfig = new RetryableSender.Config(); + + if (getErrorHandler() != null) { + senderConfig.setErrorHandler(getErrorHandler()); + } + + RetryableSender retryableSender = + new RetryableSender( + senderConfig, baseSender, new ExponentialBackOffRetryStrategy(retryStrategyConfig)); + + FluentdIngester.Config ingesterConfig = new FluentdIngester.Config(); + ingesterConfig.setAckResponseMode(isAckResponseMode()); + + return new FluentdIngester(ingesterConfig, retryableSender); + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/FluentdIngester.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/FluentdIngester.java index 23a6f5a5..fb01185a 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/FluentdIngester.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/FluentdIngester.java @@ -17,6 +17,12 @@ package org.komamitsu.fluency.fluentd.ingester; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; import org.komamitsu.fluency.fluentd.ingester.sender.FluentdSender; import org.komamitsu.fluency.fluentd.ingester.sender.RequestOption; import org.komamitsu.fluency.ingester.Ingester; @@ -25,100 +31,81 @@ import org.msgpack.core.MessagePacker; import org.msgpack.jackson.dataformat.MessagePackFactory; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; -import java.util.UUID; - -public class FluentdIngester - implements Ingester -{ - private final Config config; - private final FluentdSender sender; - private final ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - - public FluentdIngester(FluentdSender sender) - { - this(new Config(), sender); +public class FluentdIngester implements Ingester { + private final Config config; + private final FluentdSender sender; + private final ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + + public FluentdIngester(FluentdSender sender) { + this(new Config(), sender); + } + + public FluentdIngester(Config config, FluentdSender sender) { + this.config = config; + this.sender = sender; + } + + @Override + public void ingest(String tag, ByteBuffer dataBuffer) throws IOException { + ByteArrayOutputStream header = new ByteArrayOutputStream(); + MessagePacker messagePacker = MessagePack.newDefaultPacker(header); + + int dataLength = dataBuffer.limit(); + messagePacker.packArrayHeader(3); + messagePacker.packString(tag); + messagePacker.packRawStringHeader(dataLength); + messagePacker.flush(); + + ByteBuffer headerBuffer = ByteBuffer.wrap(header.toByteArray()); + + if (config.isAckResponseMode()) { + // The spec (https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#entry) + // says to encode a 128 bit value as base64, but fluent-bit currently does something different + // and in fluent-bit and fluentd there is no validation. So for now we keep it simple, see + // discussion on issue #181. + String token = UUID.randomUUID().toString(); + + ByteBuffer optionBuffer = + ByteBuffer.wrap(objectMapper.writeValueAsBytes(new RequestOption(dataLength, token))); + List buffers = Arrays.asList(headerBuffer, dataBuffer, optionBuffer); + + synchronized (sender) { + sender.sendWithAck(buffers, token); + } + } else { + ByteBuffer optionBuffer = + ByteBuffer.wrap(objectMapper.writeValueAsBytes(new RequestOption(dataLength, null))); + List buffers = Arrays.asList(headerBuffer, dataBuffer, optionBuffer); + + synchronized (sender) { + sender.send(buffers); + } } + } - public FluentdIngester(Config config, FluentdSender sender) - { - this.config = config; - this.sender = sender; - } + @Override + public Sender getSender() { + return sender; + } - @Override - public void ingest(String tag, ByteBuffer dataBuffer) - throws IOException - { - ByteArrayOutputStream header = new ByteArrayOutputStream(); - MessagePacker messagePacker = MessagePack.newDefaultPacker(header); - - int dataLength = dataBuffer.limit(); - messagePacker.packArrayHeader(3); - messagePacker.packString(tag); - messagePacker.packRawStringHeader(dataLength); - messagePacker.flush(); - - ByteBuffer headerBuffer = ByteBuffer.wrap(header.toByteArray()); - - if (config.isAckResponseMode()) { - // The spec (https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#entry) - // says to encode a 128 bit value as base64, but fluent-bit currently does something different - // and in fluent-bit and fluentd there is no validation. So for now we keep it simple, see - // discussion on issue #181. - String token = UUID.randomUUID().toString(); - - ByteBuffer optionBuffer = ByteBuffer.wrap(objectMapper.writeValueAsBytes(new RequestOption(dataLength, token))); - List buffers = Arrays.asList(headerBuffer, dataBuffer, optionBuffer); - - synchronized (sender) { - sender.sendWithAck(buffers, token); - } - } - else { - ByteBuffer optionBuffer = ByteBuffer.wrap(objectMapper.writeValueAsBytes(new RequestOption(dataLength, null))); - List buffers = Arrays.asList(headerBuffer, dataBuffer, optionBuffer); - - synchronized (sender) { - sender.send(buffers); - } - } - } + public boolean isAckResponseMode() { + return config.isAckResponseMode(); + } - @Override - public Sender getSender() - { - return sender; - } + @Override + public void close() throws IOException { + sender.close(); + } - public boolean isAckResponseMode() - { - return config.isAckResponseMode(); - } + public static class Config { + private boolean ackResponseMode = false; - @Override - public void close() - throws IOException - { - sender.close(); + public boolean isAckResponseMode() { + return ackResponseMode; } - public static class Config - { - private boolean ackResponseMode = false; - - public boolean isAckResponseMode() - { - return ackResponseMode; - } - - public void setAckResponseMode(boolean ackResponseMode) - { - this.ackResponseMode = ackResponseMode; - } + public void setAckResponseMode(boolean ackResponseMode) { + this.ackResponseMode = ackResponseMode; } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/Response.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/Response.java index 9be0b116..434e4a50 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/Response.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/Response.java @@ -18,26 +18,20 @@ import com.fasterxml.jackson.annotation.JsonProperty; -public class Response -{ - private final String ack; +public class Response { + private final String ack; - public Response(@JsonProperty("ack") String ack) - { - this.ack = ack; - } + public Response(@JsonProperty("ack") String ack) { + this.ack = ack; + } - @JsonProperty("ack") - public String getAck() - { - return ack; - } + @JsonProperty("ack") + public String getAck() { + return ack; + } - @Override - public String toString() - { - return "ResponseOption{" + - "ack=" + ack + - '}'; - } + @Override + public String toString() { + return "ResponseOption{" + "ack=" + ack + '}'; + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/FluentdSender.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/FluentdSender.java index 136de816..b2d3d438 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/FluentdSender.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/FluentdSender.java @@ -16,94 +16,76 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.komamitsu.fluency.ingester.sender.ErrorHandler; -import org.komamitsu.fluency.ingester.sender.Sender; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.komamitsu.fluency.ingester.sender.ErrorHandler; +import org.komamitsu.fluency.ingester.sender.Sender; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public abstract class FluentdSender - implements Closeable, Sender -{ - private static final Logger LOG = LoggerFactory.getLogger(FluentdSender.class); - private final Config config; +public abstract class FluentdSender implements Closeable, Sender { + private static final Logger LOG = LoggerFactory.getLogger(FluentdSender.class); + private final Config config; - protected FluentdSender(Config config) - { - this.config = config; - } + protected FluentdSender(Config config) { + this.config = config; + } - protected FluentdSender() - { - this(new FluentdSender.Config()); - } + protected FluentdSender() { + this(new FluentdSender.Config()); + } - public synchronized void send(ByteBuffer buffer) - throws IOException - { - sendInternalWithRestoreBufferPositions(Arrays.asList(buffer), null); - } + public synchronized void send(ByteBuffer buffer) throws IOException { + sendInternalWithRestoreBufferPositions(Arrays.asList(buffer), null); + } - public synchronized void send(List buffers) - throws IOException - { - sendInternalWithRestoreBufferPositions(buffers, null); - } + public synchronized void send(List buffers) throws IOException { + sendInternalWithRestoreBufferPositions(buffers, null); + } + + public void sendWithAck(List buffers, String ackToken) throws IOException { + sendInternalWithRestoreBufferPositions(buffers, ackToken); + } - public void sendWithAck(List buffers, String ackToken) - throws IOException - { - sendInternalWithRestoreBufferPositions(buffers, ackToken); + private void sendInternalWithRestoreBufferPositions(List buffers, String ackToken) + throws IOException { + List positions = new ArrayList<>(buffers.size()); + for (ByteBuffer data : buffers) { + positions.add(data.position()); } - private void sendInternalWithRestoreBufferPositions(List buffers, String ackToken) - throws IOException - { - List positions = new ArrayList<>(buffers.size()); - for (ByteBuffer data : buffers) { - positions.add(data.position()); - } + try { + sendInternal(buffers, ackToken); + } catch (Exception e) { + for (int i = 0; i < buffers.size(); i++) { + buffers.get(i).position(positions.get(i)); + } + ErrorHandler errorHandler = config.getErrorHandler(); + if (errorHandler != null) { try { - sendInternal(buffers, ackToken); + errorHandler.handle(e); + } catch (Exception ex) { + LOG.warn("Failed to handle an error in the error handler {}", errorHandler, ex); } - catch (Exception e) { - for (int i = 0; i < buffers.size(); i++) { - buffers.get(i).position(positions.get(i)); - } - - ErrorHandler errorHandler = config.getErrorHandler(); - if (errorHandler != null) { - try { - errorHandler.handle(e); - } - catch (Exception ex) { - LOG.warn("Failed to handle an error in the error handler {}", errorHandler, ex); - } - } + } - if (e instanceof IOException) { - throw (IOException) e; - } - else { - throw new IOException(e); - } - } + if (e instanceof IOException) { + throw (IOException) e; + } else { + throw new IOException(e); + } } + } - public abstract boolean isAvailable(); + public abstract boolean isAvailable(); - abstract protected void sendInternal(List buffers, String ackToken) - throws IOException; + protected abstract void sendInternal(List buffers, String ackToken) + throws IOException; - public static class Config - extends Sender.Config - { - } + public static class Config extends Sender.Config {} } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/InetSocketSender.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/InetSocketSender.java index 95a869fb..98f5c911 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/InetSocketSender.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/InetSocketSender.java @@ -20,69 +20,51 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class InetSocketSender - extends NetworkSender -{ - private static final Logger LOG = LoggerFactory.getLogger(InetSocketSender.class); - private final Config config; +public abstract class InetSocketSender extends NetworkSender { + private static final Logger LOG = LoggerFactory.getLogger(InetSocketSender.class); + private final Config config; - public InetSocketSender(Config config, FailureDetector failureDetector) - { - super(config, failureDetector); - this.config = config; - } + public InetSocketSender(Config config, FailureDetector failureDetector) { + super(config, failureDetector); + this.config = config; + } - public String getHost() - { - return config.getHost(); - } + public String getHost() { + return config.getHost(); + } - public int getPort() - { - return config.getPort(); - } + public int getPort() { + return config.getPort(); + } - @Override - public String toString() - { - return "NetworkSender{" + - "config=" + config + - "} " + super.toString(); - } + @Override + public String toString() { + return "NetworkSender{" + "config=" + config + "} " + super.toString(); + } - public static class Config - extends NetworkSender.Config - { - private String host = "127.0.0.1"; - private int port = 24224; + public static class Config extends NetworkSender.Config { + private String host = "127.0.0.1"; + private int port = 24224; - public String getHost() - { - return host; - } + public String getHost() { + return host; + } - public void setHost(String host) - { - this.host = host; - } + public void setHost(String host) { + this.host = host; + } - public int getPort() - { - return port; - } + public int getPort() { + return port; + } - public void setPort(int port) - { - this.port = port; - } + public void setPort(int port) { + this.port = port; + } - @Override - public String toString() - { - return "Config{" + - "host='" + host + '\'' + - ", port=" + port + - "} " + super.toString(); - } + @Override + public String toString() { + return "Config{" + "host='" + host + '\'' + ", port=" + port + "} " + super.toString(); } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/MultiSender.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/MultiSender.java index 278429d8..f53abc31 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/MultiSender.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/MultiSender.java @@ -16,107 +16,84 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class MultiSender - extends FluentdSender -{ - private static final Logger LOG = LoggerFactory.getLogger(MultiSender.class); - private final List senders; +public class MultiSender extends FluentdSender { + private static final Logger LOG = LoggerFactory.getLogger(MultiSender.class); + private final List senders; - public MultiSender(List senders) - { - this(new Config(), senders); - } + public MultiSender(List senders) { + this(new Config(), senders); + } - public MultiSender(Config config, List senders) - { - super(config); - this.senders = senders; - } + public MultiSender(Config config, List senders) { + super(config); + this.senders = senders; + } - @Override - public boolean isAvailable() - { - return true; - } + @Override + public boolean isAvailable() { + return true; + } - @Override - protected synchronized void sendInternal(List buffers, String ackToken) - throws AllNodesUnavailableException - { - for (FluentdSender sender : senders) { - boolean isAvailable = sender.isAvailable(); - LOG.trace("send(): sender={}, isAvailable={}", sender, isAvailable); - if (isAvailable) { - try { - if (ackToken == null) { - sender.send(buffers); - } - else { - sender.sendWithAck(buffers, ackToken); - } - return; - } - catch (IOException e) { - LOG.warn("Failed to send: sender=" + sender + ". Trying to use next sender...", e); - } - } + @Override + protected synchronized void sendInternal(List buffers, String ackToken) + throws AllNodesUnavailableException { + for (FluentdSender sender : senders) { + boolean isAvailable = sender.isAvailable(); + LOG.trace("send(): sender={}, isAvailable={}", sender, isAvailable); + if (isAvailable) { + try { + if (ackToken == null) { + sender.send(buffers); + } else { + sender.sendWithAck(buffers, ackToken); + } + return; + } catch (IOException e) { + LOG.warn("Failed to send: sender=" + sender + ". Trying to use next sender...", e); } - throw new AllNodesUnavailableException("All nodes are unavailable"); + } } + throw new AllNodesUnavailableException("All nodes are unavailable"); + } - @Override - public void close() - throws IOException - { - IOException firstException = null; - for (FluentdSender sender : senders) { - try { - sender.close(); - } - catch (IOException e) { - if (firstException == null) { - firstException = e; - } - } - } - if (firstException != null) { - throw firstException; + @Override + public void close() throws IOException { + IOException firstException = null; + for (FluentdSender sender : senders) { + try { + sender.close(); + } catch (IOException e) { + if (firstException == null) { + firstException = e; } + } } - - public List getSenders() - { - return Collections.unmodifiableList(senders); + if (firstException != null) { + throw firstException; } + } - @Override - public String toString() - { - return "MultiSender{" + - "senders=" + senders + - "} " + super.toString(); - } + public List getSenders() { + return Collections.unmodifiableList(senders); + } - public static class AllNodesUnavailableException - extends IOException - { - public AllNodesUnavailableException(String s) - { - super(s); - } - } + @Override + public String toString() { + return "MultiSender{" + "senders=" + senders + "} " + super.toString(); + } - public static class Config - extends FluentdSender.Config - { + public static class AllNodesUnavailableException extends IOException { + public AllNodesUnavailableException(String s) { + super(s); } -} + } + public static class Config extends FluentdSender.Config {} +} diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/NetworkSender.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/NetworkSender.java index 8be5d26a..79916098 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/NetworkSender.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/NetworkSender.java @@ -17,15 +17,6 @@ package org.komamitsu.fluency.fluentd.ingester.sender; import com.fasterxml.jackson.databind.ObjectMapper; -import org.komamitsu.fluency.fluentd.ingester.Response; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; -import org.komamitsu.fluency.util.ExecutorServiceUtils; -import org.komamitsu.fluency.validation.Validatable; -import org.komamitsu.fluency.validation.annotation.Min; -import org.msgpack.jackson.dataformat.MessagePackFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.SocketTimeoutException; import java.nio.ByteBuffer; @@ -35,220 +26,202 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import org.komamitsu.fluency.fluentd.ingester.Response; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; +import org.komamitsu.fluency.util.ExecutorServiceUtils; +import org.komamitsu.fluency.validation.annotation.Min; +import org.msgpack.jackson.dataformat.MessagePackFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public abstract class NetworkSender - extends FluentdSender -{ - private static final Logger LOG = LoggerFactory.getLogger(NetworkSender.class); - private final byte[] optionBuffer = new byte[256]; - private final ExecutorService executorService = ExecutorServiceUtils.newSingleThreadDaemonExecutor(); +public abstract class NetworkSender extends FluentdSender { + private static final Logger LOG = LoggerFactory.getLogger(NetworkSender.class); + private final byte[] optionBuffer = new byte[256]; + private final ExecutorService executorService = + ExecutorServiceUtils.newSingleThreadDaemonExecutor(); - private final Config config; - private final FailureDetector failureDetector; - private final ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + private final Config config; + private final FailureDetector failureDetector; + private final ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - NetworkSender(FailureDetector failureDetector) - { - this(new Config(), failureDetector); - } + NetworkSender(FailureDetector failureDetector) { + this(new Config(), failureDetector); + } - NetworkSender(Config config, FailureDetector failureDetector) - { - super(config); - this.config = config; - this.failureDetector = failureDetector; - } + NetworkSender(Config config, FailureDetector failureDetector) { + super(config); + this.config = config; + this.failureDetector = failureDetector; + } - @Override - public boolean isAvailable() - { - return failureDetector == null || failureDetector.isAvailable(); - } + @Override + public boolean isAvailable() { + return failureDetector == null || failureDetector.isAvailable(); + } - abstract T getOrCreateSocketInternal() - throws IOException; + abstract T getOrCreateSocketInternal() throws IOException; - private synchronized T getOrCreateSocket() - throws IOException - { - return getOrCreateSocketInternal(); - } + private synchronized T getOrCreateSocket() throws IOException { + return getOrCreateSocketInternal(); + } - abstract void sendBuffers(T socket, List buffers) - throws IOException; + abstract void sendBuffers(T socket, List buffers) throws IOException; - abstract void recvResponse(T socket, ByteBuffer buffer) - throws IOException; + abstract void recvResponse(T socket, ByteBuffer buffer) throws IOException; - private void propagateFailure(Throwable e) - { - if (failureDetector != null) { - failureDetector.onFailure(e); - } + private void propagateFailure(Throwable e) { + if (failureDetector != null) { + failureDetector.onFailure(e); } + } - @Override - protected synchronized void sendInternal(List buffers, String ackToken) - throws IOException - { - long totalDataSize = buffers.stream().mapToInt(ByteBuffer::remaining).sum(); + @Override + protected synchronized void sendInternal(List buffers, String ackToken) + throws IOException { + long totalDataSize = buffers.stream().mapToInt(ByteBuffer::remaining).sum(); - try { - LOG.trace("send(): sender={}, totalDataSize={}", this, totalDataSize); - final T socket = getOrCreateSocket(); - sendBuffers(socket, buffers); + try { + LOG.trace("send(): sender={}, totalDataSize={}", this, totalDataSize); + final T socket = getOrCreateSocket(); + sendBuffers(socket, buffers); - if (ackToken == null) { - return; - } + if (ackToken == null) { + return; + } - // For ACK response mode - final ByteBuffer byteBuffer = ByteBuffer.wrap(optionBuffer); + // For ACK response mode + final ByteBuffer byteBuffer = ByteBuffer.wrap(optionBuffer); - Future future = executorService.submit(() -> { + Future future = + executorService.submit( + () -> { LOG.trace("recv(): sender={}", this); recvResponse(socket, byteBuffer); return null; - }); - - try { - future.get(config.getReadTimeoutMilli(), TimeUnit.MILLISECONDS); - } - catch (InterruptedException e) { - throw new IOException("InterruptedException occurred", e); - } - catch (ExecutionException e) { - throw new IOException("ExecutionException occurred", e); - } - catch (TimeoutException e) { - throw new SocketTimeoutException("Socket read timeout"); - } - - Response response = objectMapper.readValue(optionBuffer, Response.class); - if (!ackToken.equals(response.getAck())) { - throw new UnmatchedAckException("Ack tokens don't matched: expected=" + ", got=" + response.getAck()); - } - } - catch (IOException e) { - LOG.error("Failed to send {} bytes data", totalDataSize); - closeSocket(); - propagateFailure(e); - throw e; + }); + + try { + future.get(config.getReadTimeoutMilli(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + throw new IOException("InterruptedException occurred", e); + } catch (ExecutionException e) { + throw new IOException("ExecutionException occurred", e); + } catch (TimeoutException e) { + throw new SocketTimeoutException("Socket read timeout"); + } + + Response response = objectMapper.readValue(optionBuffer, Response.class); + if (!ackToken.equals(response.getAck())) { + throw new UnmatchedAckException( + "Ack tokens don't matched: expected=" + ", got=" + response.getAck()); + } + } catch (IOException e) { + LOG.error("Failed to send {} bytes data", totalDataSize); + closeSocket(); + propagateFailure(e); + throw e; + } + } + + abstract void closeSocket() throws IOException; + + @Override + public synchronized void close() throws IOException { + try { + // Wait to confirm unsent request is flushed + try { + TimeUnit.MILLISECONDS.sleep(config.getWaitBeforeCloseMilli()); + } catch (InterruptedException e) { + LOG.warn("Interrupted", e); + Thread.currentThread().interrupt(); + } + + closeSocket(); + } finally { + try { + if (failureDetector != null) { + failureDetector.close(); } + } finally { + ExecutorServiceUtils.finishExecutorService(executorService); + } + } + } + + public int getConnectionTimeoutMilli() { + return config.getConnectionTimeoutMilli(); + } + + public int getReadTimeoutMilli() { + return config.getReadTimeoutMilli(); + } + + public FailureDetector getFailureDetector() { + return failureDetector; + } + + @Override + public String toString() { + return "NetworkSender{" + + "config=" + + config + + ", failureDetector=" + + failureDetector + + "} " + + super.toString(); + } + + public static class UnmatchedAckException extends IOException { + public UnmatchedAckException(String message) { + super(message); } + } - abstract void closeSocket() - throws IOException; + public static class Config extends FluentdSender.Config { + @Min(10) + private int connectionTimeoutMilli = 5000; - @Override - public synchronized void close() - throws IOException - { - try { - // Wait to confirm unsent request is flushed - try { - TimeUnit.MILLISECONDS.sleep(config.getWaitBeforeCloseMilli()); - } - catch (InterruptedException e) { - LOG.warn("Interrupted", e); - Thread.currentThread().interrupt(); - } - - closeSocket(); - } - finally { - try { - if (failureDetector != null) { - failureDetector.close(); - } - } - finally { - ExecutorServiceUtils.finishExecutorService(executorService); - } - } - } + @Min(10) + private int readTimeoutMilli = 5000; - public int getConnectionTimeoutMilli() - { - return config.getConnectionTimeoutMilli(); - } + @Min(0) + private int waitBeforeCloseMilli = 1000; - public int getReadTimeoutMilli() - { - return config.getReadTimeoutMilli(); + public int getConnectionTimeoutMilli() { + return connectionTimeoutMilli; } - public FailureDetector getFailureDetector() - { - return failureDetector; + public void setConnectionTimeoutMilli(int connectionTimeoutMilli) { + this.connectionTimeoutMilli = connectionTimeoutMilli; } - @Override - public String toString() - { - return "NetworkSender{" + - "config=" + config + - ", failureDetector=" + failureDetector + - "} " + super.toString(); + public int getReadTimeoutMilli() { + return readTimeoutMilli; } - public static class UnmatchedAckException - extends IOException - { - public UnmatchedAckException(String message) - { - super(message); - } + public void setReadTimeoutMilli(int readTimeoutMilli) { + this.readTimeoutMilli = readTimeoutMilli; } - public static class Config - extends FluentdSender.Config - { - @Min(10) - private int connectionTimeoutMilli = 5000; - @Min(10) - private int readTimeoutMilli = 5000; - @Min(0) - private int waitBeforeCloseMilli = 1000; - - public int getConnectionTimeoutMilli() - { - return connectionTimeoutMilli; - } - - public void setConnectionTimeoutMilli(int connectionTimeoutMilli) - { - this.connectionTimeoutMilli = connectionTimeoutMilli; - } - - public int getReadTimeoutMilli() - { - return readTimeoutMilli; - } - - public void setReadTimeoutMilli(int readTimeoutMilli) - { - this.readTimeoutMilli = readTimeoutMilli; - } - - public int getWaitBeforeCloseMilli() - { - return waitBeforeCloseMilli; - } + public int getWaitBeforeCloseMilli() { + return waitBeforeCloseMilli; + } - public void setWaitBeforeCloseMilli(int waitBeforeCloseMilli) - { - this.waitBeforeCloseMilli = waitBeforeCloseMilli; - } + public void setWaitBeforeCloseMilli(int waitBeforeCloseMilli) { + this.waitBeforeCloseMilli = waitBeforeCloseMilli; + } - @Override - public String toString() - { - return "Config{" + - "connectionTimeoutMilli=" + connectionTimeoutMilli + - ", readTimeoutMilli=" + readTimeoutMilli + - ", waitBeforeCloseMilli=" + waitBeforeCloseMilli + - "} " + super.toString(); - } + @Override + public String toString() { + return "Config{" + + "connectionTimeoutMilli=" + + connectionTimeoutMilli + + ", readTimeoutMilli=" + + readTimeoutMilli + + ", waitBeforeCloseMilli=" + + waitBeforeCloseMilli + + "} " + + super.toString(); } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/RequestOption.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/RequestOption.java index 3855a259..4ca1dd99 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/RequestOption.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/RequestOption.java @@ -20,37 +20,27 @@ import com.fasterxml.jackson.annotation.JsonProperty; @JsonInclude(JsonInclude.Include.NON_NULL) -public class RequestOption -{ - private final int size; - private final String chunk; +public class RequestOption { + private final int size; + private final String chunk; - public RequestOption( - @JsonProperty("size") int size, - @JsonProperty("chunk") String chunk) - { - this.size = size; - this.chunk = chunk; - } + public RequestOption(@JsonProperty("size") int size, @JsonProperty("chunk") String chunk) { + this.size = size; + this.chunk = chunk; + } - @JsonProperty("size") - public int getSize() - { - return size; - } + @JsonProperty("size") + public int getSize() { + return size; + } - @JsonProperty("chunk") - public String getChunk() - { - return chunk; - } + @JsonProperty("chunk") + public String getChunk() { + return chunk; + } - @Override - public String toString() - { - return "RequestOption{" + - "size=" + size + - ", chunk=" + chunk + - '}'; - } + @Override + public String toString() { + return "RequestOption{" + "size=" + size + ", chunk=" + chunk + '}'; + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/RetryableSender.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/RetryableSender.java index 68596e2f..654e4f26 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/RetryableSender.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/RetryableSender.java @@ -16,125 +16,107 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.komamitsu.fluency.fluentd.ingester.sender.retry.RetryStrategy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import org.komamitsu.fluency.fluentd.ingester.sender.retry.RetryStrategy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class RetryableSender - extends FluentdSender -{ - private static final Logger LOG = LoggerFactory.getLogger(RetryableSender.class); - private final FluentdSender baseSender; - private final AtomicBoolean isClosed = new AtomicBoolean(); - private RetryStrategy retryStrategy; - - public RetryableSender(FluentdSender baseSender, RetryStrategy retryStrategy) - { - this(new Config(), baseSender, retryStrategy); - } - - public RetryableSender(Config config, FluentdSender baseSender, RetryStrategy retryStrategy) - { - super(config); - this.baseSender = baseSender; - this.retryStrategy = retryStrategy; - } - - @Override - public void close() - throws IOException - { - baseSender.close(); - isClosed.set(true); - } - - @Override - public boolean isAvailable() - { - return true; - } - - @Override - protected synchronized void sendInternal(List buffers, String ackToken) - throws IOException - { - IOException firstException = null; - - int retry = 0; - while (!retryStrategy.isRetriedOver(retry)) { - if (isClosed.get()) { - throw new RetryOverException("This sender is already closed", firstException); - } - - try { - if (ackToken == null) { - baseSender.send(buffers); - } - else { - baseSender.sendWithAck(buffers, ackToken); - } - return; - } - catch (IOException e) { - firstException = e; - LOG.warn("Sender failed to send data. sender=" + this + ", retry=" + retry, e); - } - - try { - TimeUnit.MILLISECONDS.sleep(retryStrategy.getNextIntervalMillis(retry)); - } - catch (InterruptedException e) { - LOG.debug("Interrupted while waiting", e); - Thread.currentThread().interrupt(); - } - retry++; +public class RetryableSender extends FluentdSender { + private static final Logger LOG = LoggerFactory.getLogger(RetryableSender.class); + private final FluentdSender baseSender; + private final AtomicBoolean isClosed = new AtomicBoolean(); + private RetryStrategy retryStrategy; + + public RetryableSender(FluentdSender baseSender, RetryStrategy retryStrategy) { + this(new Config(), baseSender, retryStrategy); + } + + public RetryableSender(Config config, FluentdSender baseSender, RetryStrategy retryStrategy) { + super(config); + this.baseSender = baseSender; + this.retryStrategy = retryStrategy; + } + + @Override + public void close() throws IOException { + baseSender.close(); + isClosed.set(true); + } + + @Override + public boolean isAvailable() { + return true; + } + + @Override + protected synchronized void sendInternal(List buffers, String ackToken) + throws IOException { + IOException firstException = null; + + int retry = 0; + while (!retryStrategy.isRetriedOver(retry)) { + if (isClosed.get()) { + throw new RetryOverException("This sender is already closed", firstException); + } + + try { + if (ackToken == null) { + baseSender.send(buffers); + } else { + baseSender.sendWithAck(buffers, ackToken); } - - throw new RetryOverException("Sending data was retried over", firstException); - } - - public FluentdSender getBaseSender() - { - return baseSender; - } - - public RetryStrategy getRetryStrategy() - { - return retryStrategy; + return; + } catch (IOException e) { + firstException = e; + LOG.warn("Sender failed to send data. sender=" + this + ", retry=" + retry, e); + } + + try { + TimeUnit.MILLISECONDS.sleep(retryStrategy.getNextIntervalMillis(retry)); + } catch (InterruptedException e) { + LOG.debug("Interrupted while waiting", e); + Thread.currentThread().interrupt(); + } + retry++; } - public boolean isClosed() - { - return isClosed.get(); + throw new RetryOverException("Sending data was retried over", firstException); + } + + public FluentdSender getBaseSender() { + return baseSender; + } + + public RetryStrategy getRetryStrategy() { + return retryStrategy; + } + + public boolean isClosed() { + return isClosed.get(); + } + + @Override + public String toString() { + return "RetryableSender{" + + "baseSender=" + + baseSender + + ", retryStrategy=" + + retryStrategy + + ", isClosed=" + + isClosed + + "} " + + super.toString(); + } + + public static class RetryOverException extends IOException { + public RetryOverException(String s, Throwable throwable) { + super(s, throwable); } + } - @Override - public String toString() - { - return "RetryableSender{" + - "baseSender=" + baseSender + - ", retryStrategy=" + retryStrategy + - ", isClosed=" + isClosed + - "} " + super.toString(); - } - - public static class RetryOverException - extends IOException - { - public RetryOverException(String s, Throwable throwable) - { - super(s, throwable); - } - } - - public static class Config - extends FluentdSender.Config - { - } + public static class Config extends FluentdSender.Config {} } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSender.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSender.java index 7b319b08..39e17076 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSender.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSender.java @@ -16,14 +16,6 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; -import org.komamitsu.fluency.validation.Validatable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.net.ssl.SSLSocket; -import javax.net.ssl.SSLSocketFactory; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -31,121 +23,104 @@ import java.nio.ByteBuffer; import java.util.List; import java.util.concurrent.atomic.AtomicReference; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; +import org.komamitsu.fluency.validation.Validatable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class SSLSender - extends InetSocketSender -{ - private static final Logger LOG = LoggerFactory.getLogger(TCPSender.class); - private final AtomicReference socket = new AtomicReference<>(); - private final SSLSocketBuilder socketBuilder; - private final Config config; - - public SSLSender() - { - this(new Config()); - } - - public SSLSender(Config config) - { - this(config, null); +public class SSLSender extends InetSocketSender { + private static final Logger LOG = LoggerFactory.getLogger(TCPSender.class); + private final AtomicReference socket = new AtomicReference<>(); + private final SSLSocketBuilder socketBuilder; + private final Config config; + + public SSLSender() { + this(new Config()); + } + + public SSLSender(Config config) { + this(config, null); + } + + public SSLSender(FailureDetector failureDetector) { + this(new Config(), failureDetector); + } + + public SSLSender(Config config, FailureDetector failureDetector) { + super(config, failureDetector); + config.validateValues(); + + socketBuilder = + new SSLSocketBuilder( + config.getHost(), + config.getPort(), + config.getConnectionTimeoutMilli(), + config.getReadTimeoutMilli(), + config.getSslSocketFactory()); + this.config = config; + } + + @Override + protected SSLSocket getOrCreateSocketInternal() throws IOException { + if (socket.get() == null) { + socket.set(socketBuilder.build()); } - - public SSLSender(FailureDetector failureDetector) - { - this(new Config(), failureDetector); + return socket.get(); + } + + @Override + protected void sendBuffers(SSLSocket sslSocket, List buffers) throws IOException { + for (ByteBuffer buffer : buffers) { + OutputStream outputStream = sslSocket.getOutputStream(); + if (buffer.hasArray()) { + outputStream.write(buffer.array(), buffer.position(), buffer.remaining()); + } else { + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + outputStream.write(bytes); + } } - - public SSLSender(Config config, FailureDetector failureDetector) - { - super(config, failureDetector); - config.validateValues(); - - socketBuilder = new SSLSocketBuilder( - config.getHost(), - config.getPort(), - config.getConnectionTimeoutMilli(), - config.getReadTimeoutMilli(), - config.getSslSocketFactory()); - this.config = config; + } + + @Override + protected void recvResponse(SSLSocket sslSocket, ByteBuffer buffer) throws IOException { + InputStream inputStream = sslSocket.getInputStream(); + byte[] tempBuf = new byte[buffer.remaining()]; + int read = inputStream.read(tempBuf); + if (read < 0) { + throw new SocketException("The connection is disconnected by the peer"); } - - @Override - protected SSLSocket getOrCreateSocketInternal() - throws IOException - { - if (socket.get() == null) { - socket.set(socketBuilder.build()); - } - return socket.get(); + buffer.put(tempBuf, 0, read); + } + + @Override + protected void closeSocket() throws IOException { + SSLSocket existingSocket; + if ((existingSocket = socket.getAndSet(null)) != null) { + existingSocket.close(); } + } - @Override - protected void sendBuffers(SSLSocket sslSocket, List buffers) - throws IOException - { - for (ByteBuffer buffer : buffers) { - OutputStream outputStream = sslSocket.getOutputStream(); - if (buffer.hasArray()) { - outputStream.write(buffer.array(), buffer.position(), buffer.remaining()); - } - else { - byte[] bytes = new byte[buffer.remaining()]; - buffer.get(bytes); - outputStream.write(bytes); - } - } - } + @Override + public String toString() { + return "SSLSender{" + "config=" + config + "} " + super.toString(); + } - @Override - protected void recvResponse(SSLSocket sslSocket, ByteBuffer buffer) - throws IOException - { - InputStream inputStream = sslSocket.getInputStream(); - byte[] tempBuf = new byte[buffer.remaining()]; - int read = inputStream.read(tempBuf); - if (read < 0) { - throw new SocketException("The connection is disconnected by the peer"); - } - buffer.put(tempBuf, 0, read); + public static class Config extends InetSocketSender.Config implements Validatable { + void validateValues() { + validate(); } - @Override - protected void closeSocket() - throws IOException - { - SSLSocket existingSocket; - if ((existingSocket = socket.getAndSet(null)) != null) { - existingSocket.close(); - } - } + private SSLSocketFactory sslSocketFactory = (SSLSocketFactory) SSLSocketFactory.getDefault(); - @Override - public String toString() - { - return "SSLSender{" + - "config=" + config + - "} " + super.toString(); + public SSLSocketFactory getSslSocketFactory() { + return sslSocketFactory; } - public static class Config - extends InetSocketSender.Config - implements Validatable - { - void validateValues() - { - validate(); - } - - private SSLSocketFactory sslSocketFactory = (SSLSocketFactory) SSLSocketFactory.getDefault(); - - public SSLSocketFactory getSslSocketFactory() - { - return sslSocketFactory; - } - - public void setSslSocketFactory(SSLSocketFactory sslSocketFactory) - { - this.sslSocketFactory = sslSocketFactory; - } + public void setSslSocketFactory(SSLSocketFactory sslSocketFactory) { + this.sslSocketFactory = sslSocketFactory; } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilder.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilder.java index 97f57d10..c3257f65 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilder.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilder.java @@ -16,53 +16,48 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import javax.net.ssl.SSLSocket; -import javax.net.ssl.SSLSocketFactory; -import javax.net.SocketFactory; - import java.io.IOException; import java.net.InetSocketAddress; import java.net.Socket; +import javax.net.SocketFactory; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; -public class SSLSocketBuilder -{ - private final String host; - private final int port; - private final int connectionTimeoutMilli; - private final int readTimeoutMilli; - private final SSLSocketFactory sslSocketFactory; +public class SSLSocketBuilder { + private final String host; + private final int port; + private final int connectionTimeoutMilli; + private final int readTimeoutMilli; + private final SSLSocketFactory sslSocketFactory; - public SSLSocketBuilder(String host, Integer port, int connectionTimeoutMilli, int readTimeoutMilli, SocketFactory sslSocketFactory) - { - this.host = host; - this.port = port; - this.connectionTimeoutMilli = connectionTimeoutMilli; - this.readTimeoutMilli = readTimeoutMilli; - this.sslSocketFactory = (SSLSocketFactory)sslSocketFactory; - } + public SSLSocketBuilder( + String host, + Integer port, + int connectionTimeoutMilli, + int readTimeoutMilli, + SocketFactory sslSocketFactory) { + this.host = host; + this.port = port; + this.connectionTimeoutMilli = connectionTimeoutMilli; + this.readTimeoutMilli = readTimeoutMilli; + this.sslSocketFactory = (SSLSocketFactory) sslSocketFactory; + } - public SSLSocket build() - throws IOException - { - Socket socket = this.sslSocketFactory.createSocket(); - try { - socket.connect(new InetSocketAddress(host, port), connectionTimeoutMilli); - socket.setTcpNoDelay(true); - socket.setSoTimeout(readTimeoutMilli); - } - catch (Throwable e) { - socket.close(); - throw e; - } - return (SSLSocket) socket; + public SSLSocket build() throws IOException { + Socket socket = this.sslSocketFactory.createSocket(); + try { + socket.connect(new InetSocketAddress(host, port), connectionTimeoutMilli); + socket.setTcpNoDelay(true); + socket.setSoTimeout(readTimeoutMilli); + } catch (Throwable e) { + socket.close(); + throw e; } + return (SSLSocket) socket; + } - @Override - public String toString() - { - return "SSLSocketBuilder{" + - "host='" + host + '\'' + - ", port=" + port + - '}'; - } + @Override + public String toString() { + return "SSLSocketBuilder{" + "host='" + host + '\'' + ", port=" + port + '}'; + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/TCPSender.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/TCPSender.java index 1eb99fb4..90b6310b 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/TCPSender.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/TCPSender.java @@ -16,11 +16,6 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; -import org.komamitsu.fluency.validation.Validatable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketException; @@ -28,100 +23,87 @@ import java.nio.channels.SocketChannel; import java.util.List; import java.util.concurrent.atomic.AtomicReference; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; +import org.komamitsu.fluency.validation.Validatable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class TCPSender - extends InetSocketSender -{ - private static final Logger LOG = LoggerFactory.getLogger(TCPSender.class); - private final AtomicReference channel = new AtomicReference<>(); - private final Config config; +public class TCPSender extends InetSocketSender { + private static final Logger LOG = LoggerFactory.getLogger(TCPSender.class); + private final AtomicReference channel = new AtomicReference<>(); + private final Config config; - public TCPSender() - { - this(new Config()); - } + public TCPSender() { + this(new Config()); + } - public TCPSender(Config config) - { - this(config, null); - } + public TCPSender(Config config) { + this(config, null); + } - public TCPSender(FailureDetector failureDetector) - { - this(new Config(), failureDetector); - } + public TCPSender(FailureDetector failureDetector) { + this(new Config(), failureDetector); + } - public TCPSender(Config config, FailureDetector failureDetector) - { - super(config, failureDetector); - config.validateValues(); - this.config = config; - } + public TCPSender(Config config, FailureDetector failureDetector) { + super(config, failureDetector); + config.validateValues(); + this.config = config; + } - @Override - protected SocketChannel getOrCreateSocketInternal() - throws IOException - { - if (channel.get() == null) { - SocketChannel socketChannel = SocketChannel.open(); - try { - socketChannel.socket().connect(new InetSocketAddress(config.getHost(), config.getPort()), config.getConnectionTimeoutMilli()); - socketChannel.socket().setTcpNoDelay(true); - socketChannel.socket().setSoTimeout(config.getReadTimeoutMilli()); - } - catch (Throwable e) { - // In case of java.net.UnknownHostException and so on, the internal socket can be leaked. - // So the SocketChannel should be closed here to avoid a socket leak. - socketChannel.close(); - throw e; - } - channel.set(socketChannel); - } - return channel.get(); + @Override + protected SocketChannel getOrCreateSocketInternal() throws IOException { + if (channel.get() == null) { + SocketChannel socketChannel = SocketChannel.open(); + try { + socketChannel + .socket() + .connect( + new InetSocketAddress(config.getHost(), config.getPort()), + config.getConnectionTimeoutMilli()); + socketChannel.socket().setTcpNoDelay(true); + socketChannel.socket().setSoTimeout(config.getReadTimeoutMilli()); + } catch (Throwable e) { + // In case of java.net.UnknownHostException and so on, the internal socket can be leaked. + // So the SocketChannel should be closed here to avoid a socket leak. + socketChannel.close(); + throw e; + } + channel.set(socketChannel); } + return channel.get(); + } - @Override - protected void sendBuffers(SocketChannel socketChannel, List buffers) - throws IOException - { - socketChannel.write(buffers.toArray(new ByteBuffer[0])); - } + @Override + protected void sendBuffers(SocketChannel socketChannel, List buffers) + throws IOException { + socketChannel.write(buffers.toArray(new ByteBuffer[0])); + } - @Override - protected void recvResponse(SocketChannel socketChannel, ByteBuffer buffer) - throws IOException - { - int read = socketChannel.read(buffer); - if (read < 0) { - throw new SocketException("The connection is disconnected by the peer"); - } + @Override + protected void recvResponse(SocketChannel socketChannel, ByteBuffer buffer) throws IOException { + int read = socketChannel.read(buffer); + if (read < 0) { + throw new SocketException("The connection is disconnected by the peer"); } + } - @Override - protected void closeSocket() - throws IOException - { - SocketChannel existingSocketChannel; - if ((existingSocketChannel = channel.getAndSet(null)) != null) { - existingSocketChannel.close(); - } + @Override + protected void closeSocket() throws IOException { + SocketChannel existingSocketChannel; + if ((existingSocketChannel = channel.getAndSet(null)) != null) { + existingSocketChannel.close(); } + } - @Override - public String toString() - { - return "TCPSender{" + - "config=" + config + - "} " + super.toString(); - } + @Override + public String toString() { + return "TCPSender{" + "config=" + config + "} " + super.toString(); + } - public static class Config - extends InetSocketSender.Config - implements Validatable - { - void validateValues() - { - validate(); - } + public static class Config extends InetSocketSender.Config implements Validatable { + void validateValues() { + validate(); } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetectStrategy.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetectStrategy.java index 63b69c89..749d478f 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetectStrategy.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetectStrategy.java @@ -16,20 +16,16 @@ package org.komamitsu.fluency.fluentd.ingester.sender.failuredetect; -public abstract class FailureDetectStrategy -{ - protected final Config config; +public abstract class FailureDetectStrategy { + protected final Config config; - protected FailureDetectStrategy(Config config) - { - this.config = config; - } + protected FailureDetectStrategy(Config config) { + this.config = config; + } - public abstract void heartbeat(long now); + public abstract void heartbeat(long now); - public abstract boolean isAvailable(); + public abstract boolean isAvailable(); - public static class Config - { - } + public static class Config {} } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetector.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetector.java index ecaa4235..880f40e3 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetector.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetector.java @@ -16,121 +16,104 @@ package org.komamitsu.fluency.fluentd.ingester.sender.failuredetect; +import java.io.Closeable; +import java.util.concurrent.atomic.AtomicReference; import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.Heartbeater; import org.komamitsu.fluency.validation.Validatable; import org.komamitsu.fluency.validation.annotation.Min; -import java.io.Closeable; -import java.util.concurrent.atomic.AtomicReference; - -public class FailureDetector - implements Heartbeater.Callback, Closeable -{ - private final FailureDetectStrategy failureDetectStrategy; - private final Heartbeater heartbeater; - private final AtomicReference lastFailureTimestampMillis = new AtomicReference<>(); - private final Config config; - - public FailureDetector(FailureDetectStrategy failureDetectStrategy, Heartbeater heartbeater, Config config) - { - config.validateValues(); - this.failureDetectStrategy = failureDetectStrategy; - this.heartbeater = heartbeater; - this.heartbeater.setCallback(this); - this.heartbeater.start(); - this.config = config; +public class FailureDetector implements Heartbeater.Callback, Closeable { + private final FailureDetectStrategy failureDetectStrategy; + private final Heartbeater heartbeater; + private final AtomicReference lastFailureTimestampMillis = new AtomicReference<>(); + private final Config config; + + public FailureDetector( + FailureDetectStrategy failureDetectStrategy, Heartbeater heartbeater, Config config) { + config.validateValues(); + this.failureDetectStrategy = failureDetectStrategy; + this.heartbeater = heartbeater; + this.heartbeater.setCallback(this); + this.heartbeater.start(); + this.config = config; + } + + public FailureDetector(FailureDetectStrategy failureDetectStrategy, Heartbeater heartbeater) { + this(failureDetectStrategy, heartbeater, new Config()); + } + + @Override + public void onHeartbeat() { + failureDetectStrategy.heartbeat(System.currentTimeMillis()); + } + + @Override + public void onFailure(Throwable cause) { + lastFailureTimestampMillis.set(System.currentTimeMillis()); + } + + public boolean isAvailable() { + Long failureTimestamp = lastFailureTimestampMillis.get(); + if (failureTimestamp != null) { + if (failureTimestamp > System.currentTimeMillis() - config.getFailureIntervalMillis()) { + return false; + } else { + lastFailureTimestampMillis.set(null); + } } - - public FailureDetector(FailureDetectStrategy failureDetectStrategy, Heartbeater heartbeater) - { - this(failureDetectStrategy, heartbeater, new Config()); + return failureDetectStrategy.isAvailable(); + } + + @Override + public void close() { + heartbeater.close(); + } + + public FailureDetectStrategy getFailureDetectStrategy() { + return failureDetectStrategy; + } + + public Heartbeater getHeartbeater() { + return heartbeater; + } + + public int getFailureIntervalMillis() { + return config.getFailureIntervalMillis(); + } + + @Override + public String toString() { + return "FailureDetector{" + + "failureDetectStrategy=" + + failureDetectStrategy + + ", heartbeater=" + + heartbeater + + ", lastFailureTimestampMillis=" + + lastFailureTimestampMillis + + ", config=" + + config + + '}'; + } + + public static class Config implements Validatable { + @Min(0) + private int failureIntervalMillis = 3 * 1000; + + public int getFailureIntervalMillis() { + return failureIntervalMillis; } - @Override - public void onHeartbeat() - { - failureDetectStrategy.heartbeat(System.currentTimeMillis()); + public void setFailureIntervalMillis(int failureIntervalMillis) { + this.failureIntervalMillis = failureIntervalMillis; } - @Override - public void onFailure(Throwable cause) - { - lastFailureTimestampMillis.set(System.currentTimeMillis()); - } - - public boolean isAvailable() - { - Long failureTimestamp = lastFailureTimestampMillis.get(); - if (failureTimestamp != null) { - if (failureTimestamp > System.currentTimeMillis() - config.getFailureIntervalMillis()) { - return false; - } - else { - lastFailureTimestampMillis.set(null); - } - } - return failureDetectStrategy.isAvailable(); + void validateValues() { + validate(); } @Override - public void close() - { - heartbeater.close(); - } - - public FailureDetectStrategy getFailureDetectStrategy() - { - return failureDetectStrategy; - } - - public Heartbeater getHeartbeater() - { - return heartbeater; - } - - public int getFailureIntervalMillis() - { - return config.getFailureIntervalMillis(); - } - - @Override - public String toString() - { - return "FailureDetector{" + - "failureDetectStrategy=" + failureDetectStrategy + - ", heartbeater=" + heartbeater + - ", lastFailureTimestampMillis=" + lastFailureTimestampMillis + - ", config=" + config + - '}'; - } - - public static class Config - implements Validatable - { - @Min(0) - private int failureIntervalMillis = 3 * 1000; - - public int getFailureIntervalMillis() - { - return failureIntervalMillis; - } - - public void setFailureIntervalMillis(int failureIntervalMillis) - { - this.failureIntervalMillis = failureIntervalMillis; - } - - void validateValues() - { - validate(); - } - - @Override - public String toString() - { - return "Config{" + - "failureIntervalMillis=" + failureIntervalMillis + - '}'; - } + public String toString() { + return "Config{" + "failureIntervalMillis=" + failureIntervalMillis + '}'; } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/PhiAccrualFailureDetectStrategy.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/PhiAccrualFailureDetectStrategy.java index 064b0891..b93ef3e0 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/PhiAccrualFailureDetectStrategy.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/PhiAccrualFailureDetectStrategy.java @@ -21,90 +21,77 @@ import org.komamitsu.fluency.validation.annotation.DecimalMin; import org.komamitsu.fluency.validation.annotation.Min; -public class PhiAccrualFailureDetectStrategy - extends FailureDetectStrategy -{ - private final PhiAccuralFailureDetector failureDetector; - private final Config config; - - public PhiAccrualFailureDetectStrategy() - { - this(new Config()); +public class PhiAccrualFailureDetectStrategy extends FailureDetectStrategy { + private final PhiAccuralFailureDetector failureDetector; + private final Config config; + + public PhiAccrualFailureDetectStrategy() { + this(new Config()); + } + + public PhiAccrualFailureDetectStrategy(Config config) { + super(config); + config.validateValues(); + this.config = config; + failureDetector = + new PhiAccuralFailureDetector.Builder() + .setThreshold(config.getPhiThreshold()) + .setMaxSampleSize(config.getArrivalWindowSize()) + .build(); + } + + @Override + public void heartbeat(long now) { + failureDetector.heartbeat(now); + } + + @Override + public boolean isAvailable() { + return failureDetector.isAvailable(); + } + + public double getPhiThreshold() { + return config.getPhiThreshold(); + } + + public int getArrivalWindowSize() { + return config.getArrivalWindowSize(); + } + + @Override + public String toString() { + return "PhiAccrualFailureDetectStrategy{" + + "failureDetector=" + + failureDetector + + "} " + + super.toString(); + } + + public static class Config extends FailureDetectStrategy.Config implements Validatable { + @DecimalMin(value = "0", inclusive = false) + private double phiThreshold = 16; + + @Min(value = 0, inclusive = false) + private int arrivalWindowSize = 100; + + public double getPhiThreshold() { + return phiThreshold; } - public PhiAccrualFailureDetectStrategy(Config config) - { - super(config); - config.validateValues(); - this.config = config; - failureDetector = new PhiAccuralFailureDetector.Builder(). - setThreshold(config.getPhiThreshold()). - setMaxSampleSize(config.getArrivalWindowSize()). - build(); + public void setPhiThreshold(float phiThreshold) { + this.phiThreshold = phiThreshold; } - @Override - public void heartbeat(long now) - { - failureDetector.heartbeat(now); + public int getArrivalWindowSize() { + return arrivalWindowSize; } - @Override - public boolean isAvailable() - { - return failureDetector.isAvailable(); + void validateValues() { + validate(); } - public double getPhiThreshold() - { - return config.getPhiThreshold(); - } - - public int getArrivalWindowSize() - { - return config.getArrivalWindowSize(); - } - - @Override - public String toString() - { - return "PhiAccrualFailureDetectStrategy{" + - "failureDetector=" + failureDetector + - "} " + super.toString(); - } - - public static class Config - extends FailureDetectStrategy.Config - implements Validatable - { - @DecimalMin(value = "0", inclusive = false) - private double phiThreshold = 16; - @Min(value = 0, inclusive = false) - private int arrivalWindowSize = 100; - - public double getPhiThreshold() - { - return phiThreshold; - } - - public void setPhiThreshold(float phiThreshold) - { - this.phiThreshold = phiThreshold; - } - - public int getArrivalWindowSize() - { - return arrivalWindowSize; - } - - void validateValues() - { - validate(); - } - - public void setArrivalWindowSize(int arrivalWindowSize) - { - this.arrivalWindowSize = arrivalWindowSize; - } + public void setArrivalWindowSize(int arrivalWindowSize) { + this.arrivalWindowSize = arrivalWindowSize; } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/Heartbeater.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/Heartbeater.java index 1f025834..196e55e6 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/Heartbeater.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/Heartbeater.java @@ -16,125 +16,98 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.komamitsu.fluency.util.ExecutorServiceUtils; -import org.komamitsu.fluency.validation.Validatable; -import org.komamitsu.fluency.validation.annotation.Min; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.IOException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import org.komamitsu.fluency.util.ExecutorServiceUtils; +import org.komamitsu.fluency.validation.Validatable; +import org.komamitsu.fluency.validation.annotation.Min; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public abstract class Heartbeater - implements Closeable -{ - private static final Logger LOG = LoggerFactory.getLogger(Heartbeater.class); - private final Config config; - private final ScheduledExecutorService executorService; - private final AtomicReference callback = new AtomicReference<>(); - - protected Heartbeater(Config config) - { - config.validateValues(); - this.config = config; - executorService = ExecutorServiceUtils.newScheduledDaemonThreadPool(1); +public abstract class Heartbeater implements Closeable { + private static final Logger LOG = LoggerFactory.getLogger(Heartbeater.class); + private final Config config; + private final ScheduledExecutorService executorService; + private final AtomicReference callback = new AtomicReference<>(); + + protected Heartbeater(Config config) { + config.validateValues(); + this.config = config; + executorService = ExecutorServiceUtils.newScheduledDaemonThreadPool(1); + } + + public void start() { + executorService.scheduleAtFixedRate( + this::ping, config.getIntervalMillis(), config.getIntervalMillis(), TimeUnit.MILLISECONDS); + } + + protected abstract void invoke() throws IOException; + + protected void ping() { + try { + invoke(); + } catch (Throwable e) { + LOG.warn("ping(): failed, config=" + config); + Callback callback = this.callback.get(); + if (callback != null) { + callback.onFailure(e); + } } + } - public void start() - { - executorService.scheduleAtFixedRate( - this::ping, - config.getIntervalMillis(), - config.getIntervalMillis(), - TimeUnit.MILLISECONDS); + protected void pong() { + Callback callback = this.callback.get(); + if (callback != null) { + callback.onHeartbeat(); } + } - protected abstract void invoke() - throws IOException; - - protected void ping() - { - try { - invoke(); - } - catch (Throwable e) { - LOG.warn("ping(): failed, config=" + config); - Callback callback = this.callback.get(); - if (callback != null) { - callback.onFailure(e); - } - } - } + public void setCallback(Callback callback) { + this.callback.set(callback); + } - protected void pong() - { - Callback callback = this.callback.get(); - if (callback != null) { - callback.onHeartbeat(); - } - } + @Override + public void close() { + ExecutorServiceUtils.finishExecutorService(executorService); + } - public void setCallback(Callback callback) - { - this.callback.set(callback); - } + public int getIntervalMillis() { + return config.getIntervalMillis(); + } - @Override - public void close() - { - ExecutorServiceUtils.finishExecutorService(executorService); - } + @Override + public String toString() { + return "Heartbeater{" + "config=" + config + '}'; + } - public int getIntervalMillis() - { - return config.getIntervalMillis(); - } + public interface Callback { + void onHeartbeat(); - @Override - public String toString() - { - return "Heartbeater{" + - "config=" + config + - '}'; + void onFailure(Throwable cause); + } + + public static class Config implements Validatable { + @Min(100) + private int intervalMillis = 1000; + + public int getIntervalMillis() { + return intervalMillis; } - public interface Callback - { - void onHeartbeat(); + public void setIntervalMillis(int intervalMillis) { + this.intervalMillis = intervalMillis; + } - void onFailure(Throwable cause); + void validateValues() { + validate(); } - public static class Config - implements Validatable - { - @Min(100) - private int intervalMillis = 1000; - - public int getIntervalMillis() - { - return intervalMillis; - } - - public void setIntervalMillis(int intervalMillis) - { - this.intervalMillis = intervalMillis; - } - - void validateValues() - { - validate(); - } - - @Override - public String toString() - { - return "Config{" + - "intervalMillis=" + intervalMillis + - '}'; - } + @Override + public String toString() { + return "Config{" + "intervalMillis=" + intervalMillis + '}'; } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/InetSocketHeartbeater.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/InetSocketHeartbeater.java index bd288d55..6e60da28 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/InetSocketHeartbeater.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/InetSocketHeartbeater.java @@ -16,88 +16,67 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; +import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; +public abstract class InetSocketHeartbeater extends Heartbeater { + private static final Logger LOG = LoggerFactory.getLogger(InetSocketHeartbeater.class); + private final Config config; -public abstract class InetSocketHeartbeater - extends Heartbeater -{ - private static final Logger LOG = LoggerFactory.getLogger(InetSocketHeartbeater.class); - private final Config config; + protected InetSocketHeartbeater(Config config) { + super(config); + this.config = config; + } - protected InetSocketHeartbeater(Config config) - { - super(config); - this.config = config; - } + protected abstract void invoke() throws IOException; - protected abstract void invoke() - throws IOException; + public String getHost() { + return config.getHost(); + } - public String getHost() - { - return config.getHost(); - } + public int getPort() { + return config.getPort(); + } - public int getPort() - { - return config.getPort(); - } + public int getIntervalMillis() { + return config.getIntervalMillis(); + } + + @Override + public String toString() { + return "InetHeartbeater{" + "config=" + config + '}'; + } - public int getIntervalMillis() - { - return config.getIntervalMillis(); + public interface Callback { + void onHeartbeat(); + + void onFailure(Throwable cause); + } + + public static class Config extends Heartbeater.Config { + private String host = "127.0.0.1"; + private int port = 24224; + + public String getHost() { + return host; } - @Override - public String toString() - { - return "InetHeartbeater{" + - "config=" + config + - '}'; + public void setHost(String host) { + this.host = host; } - public interface Callback - { - void onHeartbeat(); + public int getPort() { + return port; + } - void onFailure(Throwable cause); + public void setPort(int port) { + this.port = port; } - public static class Config - extends Heartbeater.Config - { - private String host = "127.0.0.1"; - private int port = 24224; - - public String getHost() - { - return host; - } - - public void setHost(String host) - { - this.host = host; - } - - public int getPort() - { - return port; - } - - public void setPort(int port) - { - this.port = port; - } - - @Override - public String toString() { - return "Config{" + - "host='" + host + '\'' + - ", port=" + port + - "} " + super.toString(); - } + @Override + public String toString() { + return "Config{" + "host='" + host + '\'' + ", port=" + port + "} " + super.toString(); } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeater.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeater.java index e8cd6a0e..8931be09 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeater.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeater.java @@ -16,109 +16,96 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; +import java.io.IOException; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; import org.komamitsu.fluency.fluentd.ingester.sender.SSLSocketBuilder; import org.komamitsu.fluency.validation.Validatable; import org.komamitsu.fluency.validation.annotation.Min; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.net.ssl.SSLSocket; -import javax.net.ssl.SSLSocketFactory; - -import java.io.IOException; - -public class SSLHeartbeater - extends InetSocketHeartbeater -{ - private static final Logger LOG = LoggerFactory.getLogger(SSLHeartbeater.class); - private final Config config; - private final SSLSocketBuilder sslSocketBuilder; - - public SSLHeartbeater() - { - this(new Config()); - } - - public SSLHeartbeater(final Config config) - { - super(config); - config.validateValues(); - this.config = config; - sslSocketBuilder = new SSLSocketBuilder( +public class SSLHeartbeater extends InetSocketHeartbeater { + private static final Logger LOG = LoggerFactory.getLogger(SSLHeartbeater.class); + private final Config config; + private final SSLSocketBuilder sslSocketBuilder; + + public SSLHeartbeater() { + this(new Config()); + } + + public SSLHeartbeater(final Config config) { + super(config); + config.validateValues(); + this.config = config; + sslSocketBuilder = + new SSLSocketBuilder( config.getHost(), config.getPort(), config.connectionTimeoutMilli, config.readTimeoutMilli, - config.getSslSocketFactory() - ); + config.getSslSocketFactory()); + } + + @Override + protected void invoke() throws IOException { + try (SSLSocket sslSocket = sslSocketBuilder.build()) { + LOG.trace( + "SSLHeartbeat: remotePort={}, localPort={}", + sslSocket.getPort(), + sslSocket.getLocalPort()); + // Try SSL handshake + sslSocket.getSession(); + pong(); + } + } + + @Override + public String toString() { + return "SSLHeartbeater{" + + "config=" + + config + + ", sslSocketBuilder=" + + sslSocketBuilder + + "} " + + super.toString(); + } + + public static class Config extends InetSocketHeartbeater.Config implements Validatable { + @Min(10) + private int connectionTimeoutMilli = 5000; + + @Min(10) + private int readTimeoutMilli = 5000; + + public int getConnectionTimeoutMilli() { + return connectionTimeoutMilli; } - @Override - protected void invoke() - throws IOException - { - try (SSLSocket sslSocket = sslSocketBuilder.build()) { - LOG.trace("SSLHeartbeat: remotePort={}, localPort={}", sslSocket.getPort(), sslSocket.getLocalPort()); - // Try SSL handshake - sslSocket.getSession(); - pong(); - } + public void setConnectionTimeoutMilli(int connectionTimeoutMilli) { + this.connectionTimeoutMilli = connectionTimeoutMilli; } - @Override - public String toString() - { - return "SSLHeartbeater{" + - "config=" + config + - ", sslSocketBuilder=" + sslSocketBuilder + - "} " + super.toString(); + public int getReadTimeoutMilli() { + return readTimeoutMilli; } - public static class Config - extends InetSocketHeartbeater.Config - implements Validatable - { - @Min(10) - private int connectionTimeoutMilli = 5000; - @Min(10) - private int readTimeoutMilli = 5000; - - public int getConnectionTimeoutMilli() - { - return connectionTimeoutMilli; - } - - public void setConnectionTimeoutMilli(int connectionTimeoutMilli) - { - this.connectionTimeoutMilli = connectionTimeoutMilli; - } - - public int getReadTimeoutMilli() - { - return readTimeoutMilli; - } - - public void setReadTimeoutMilli(int readTimeoutMilli) - { - this.readTimeoutMilli = readTimeoutMilli; - } - - private SSLSocketFactory sslSocketFactory = (SSLSocketFactory) SSLSocketFactory.getDefault(); - - public SSLSocketFactory getSslSocketFactory() - { - return sslSocketFactory; - } - - public void setSslSocketFactory(SSLSocketFactory sslSocketFactory) - { - this.sslSocketFactory = sslSocketFactory; - } - - void validateValues() - { - validate(); - } + public void setReadTimeoutMilli(int readTimeoutMilli) { + this.readTimeoutMilli = readTimeoutMilli; + } + + private SSLSocketFactory sslSocketFactory = (SSLSocketFactory) SSLSocketFactory.getDefault(); + + public SSLSocketFactory getSslSocketFactory() { + return sslSocketFactory; + } + + public void setSslSocketFactory(SSLSocketFactory sslSocketFactory) { + this.sslSocketFactory = sslSocketFactory; } -} + void validateValues() { + validate(); + } + } +} diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/TCPHeartbeater.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/TCPHeartbeater.java index 3f1c73a1..87003f45 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/TCPHeartbeater.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/TCPHeartbeater.java @@ -16,52 +16,41 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.InetSocketAddress; import java.nio.channels.SocketChannel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class TCPHeartbeater - extends InetSocketHeartbeater -{ - private static final Logger LOG = LoggerFactory.getLogger(TCPHeartbeater.class); - private final Config config; - - public TCPHeartbeater() - { - this(new Config()); - } - - public TCPHeartbeater(final Config config) - { - super(config); - this.config = config; - } - - @Override - protected void invoke() - throws IOException - { - try (SocketChannel socketChannel = SocketChannel.open(new InetSocketAddress(config.getHost(), config.getPort()))) { - LOG.trace("TCPHeartbeat: remotePort={}, localPort={}", - socketChannel.socket().getPort(), socketChannel.socket().getLocalPort()); - pong(); - } +public class TCPHeartbeater extends InetSocketHeartbeater { + private static final Logger LOG = LoggerFactory.getLogger(TCPHeartbeater.class); + private final Config config; + + public TCPHeartbeater() { + this(new Config()); + } + + public TCPHeartbeater(final Config config) { + super(config); + this.config = config; + } + + @Override + protected void invoke() throws IOException { + try (SocketChannel socketChannel = + SocketChannel.open(new InetSocketAddress(config.getHost(), config.getPort()))) { + LOG.trace( + "TCPHeartbeat: remotePort={}, localPort={}", + socketChannel.socket().getPort(), + socketChannel.socket().getLocalPort()); + pong(); } + } - @Override - public String toString() - { - return "TCPHeartbeater{" + - "config=" + config + - "} " + super.toString(); - } + @Override + public String toString() { + return "TCPHeartbeater{" + "config=" + config + "} " + super.toString(); + } - public static class Config - extends InetSocketHeartbeater.Config - { - } + public static class Config extends InetSocketHeartbeater.Config {} } - diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UDPHeartbeater.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UDPHeartbeater.java index 0af77b53..1e79200c 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UDPHeartbeater.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UDPHeartbeater.java @@ -16,54 +16,41 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.nio.ByteBuffer; import java.nio.channels.DatagramChannel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class UDPHeartbeater - extends InetSocketHeartbeater -{ - private static final Logger LOG = LoggerFactory.getLogger(UDPHeartbeater.class); - private final SocketAddress socketAddress; - - public UDPHeartbeater() - { - this(new Config()); - } - - public UDPHeartbeater(final Config config) - { - super(config); - socketAddress = new InetSocketAddress(config.getHost(), config.getPort()); - } - - @Override - protected void invoke() - throws IOException - { - try (DatagramChannel datagramChannel = DatagramChannel.open()) { - ByteBuffer byteBuffer = ByteBuffer.allocate(0); - datagramChannel.send(byteBuffer, socketAddress); - datagramChannel.receive(byteBuffer); - pong(); - } +public class UDPHeartbeater extends InetSocketHeartbeater { + private static final Logger LOG = LoggerFactory.getLogger(UDPHeartbeater.class); + private final SocketAddress socketAddress; + + public UDPHeartbeater() { + this(new Config()); + } + + public UDPHeartbeater(final Config config) { + super(config); + socketAddress = new InetSocketAddress(config.getHost(), config.getPort()); + } + + @Override + protected void invoke() throws IOException { + try (DatagramChannel datagramChannel = DatagramChannel.open()) { + ByteBuffer byteBuffer = ByteBuffer.allocate(0); + datagramChannel.send(byteBuffer, socketAddress); + datagramChannel.receive(byteBuffer); + pong(); } + } - @Override - public String toString() - { - return "UDPHeartbeater{" + - "socketAddress=" + socketAddress + - "} " + super.toString(); - } + @Override + public String toString() { + return "UDPHeartbeater{" + "socketAddress=" + socketAddress + "} " + super.toString(); + } - public static class Config - extends InetSocketHeartbeater.Config - { - } + public static class Config extends InetSocketHeartbeater.Config {} } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ConstantRetryStrategy.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ConstantRetryStrategy.java index 08c2ce0c..c0be6ca6 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ConstantRetryStrategy.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ConstantRetryStrategy.java @@ -19,60 +19,44 @@ import org.komamitsu.fluency.validation.Validatable; import org.komamitsu.fluency.validation.annotation.Min; -public class ConstantRetryStrategy - extends RetryStrategy -{ - private final Config config; - - public ConstantRetryStrategy(Config config) - { - super(config); - config.validateValues(); - this.config = config; +public class ConstantRetryStrategy extends RetryStrategy { + private final Config config; + + public ConstantRetryStrategy(Config config) { + super(config); + config.validateValues(); + this.config = config; + } + + @Override + public int getNextIntervalMillis(int retryCount) { + return config.getRetryIntervalMillis(); + } + + @Override + public String toString() { + return "ConstantRetryStrategy{" + "config=" + config + "} " + super.toString(); + } + + public static class Config extends RetryStrategy.Config implements Validatable { + @Min(10) + private int retryIntervalMillis = 2000; + + public int getRetryIntervalMillis() { + return retryIntervalMillis; } - @Override - public int getNextIntervalMillis(int retryCount) - { - return config.getRetryIntervalMillis(); + public void setRetryIntervalMillis(int retryIntervalMillis) { + this.retryIntervalMillis = retryIntervalMillis; } - @Override - public String toString() - { - return "ConstantRetryStrategy{" + - "config=" + config + - "} " + super.toString(); + void validateValues() { + validate(); } - public static class Config - extends RetryStrategy.Config - implements Validatable - { - @Min(10) - private int retryIntervalMillis = 2000; - - public int getRetryIntervalMillis() - { - return retryIntervalMillis; - } - - public void setRetryIntervalMillis(int retryIntervalMillis) - { - this.retryIntervalMillis = retryIntervalMillis; - } - - void validateValues() - { - validate(); - } - - @Override - public String toString() - { - return "Config{" + - "retryIntervalMillis=" + retryIntervalMillis + - "} " + super.toString(); - } + @Override + public String toString() { + return "Config{" + "retryIntervalMillis=" + retryIntervalMillis + "} " + super.toString(); } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ExponentialBackOffRetryStrategy.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ExponentialBackOffRetryStrategy.java index a8f03c60..920f0995 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ExponentialBackOffRetryStrategy.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ExponentialBackOffRetryStrategy.java @@ -19,92 +19,77 @@ import org.komamitsu.fluency.validation.Validatable; import org.komamitsu.fluency.validation.annotation.Min; -public class ExponentialBackOffRetryStrategy - extends RetryStrategy -{ - private final Config config; - - public ExponentialBackOffRetryStrategy() - { - this(new Config()); +public class ExponentialBackOffRetryStrategy extends RetryStrategy { + private final Config config; + + public ExponentialBackOffRetryStrategy() { + this(new Config()); + } + + public ExponentialBackOffRetryStrategy(Config config) { + super(config); + config.validateValues(); + this.config = config; + } + + @Override + public int getNextIntervalMillis(int retryCount) { + int interval = config.getBaseIntervalMillis() * ((int) Math.pow(2.0, (double) retryCount)); + if (interval > config.getMaxIntervalMillis()) { + return config.getMaxIntervalMillis(); } + return interval; + } - public ExponentialBackOffRetryStrategy(Config config) - { - super(config); - config.validateValues(); - this.config = config; + public int getBaseIntervalMillis() { + return config.getBaseIntervalMillis(); + } + + public int getMaxIntervalMillis() { + return config.getMaxIntervalMillis(); + } + + @Override + public String toString() { + return "ExponentialBackOffRetryStrategy{" + "config=" + config + "} " + super.toString(); + } + + public static class Config extends RetryStrategy.Config implements Validatable { + @Min(10) + private int baseIntervalMillis = 400; + + @Min(10) + private int maxIntervalMillis = 30 * 1000; + + public int getBaseIntervalMillis() { + return baseIntervalMillis; } - @Override - public int getNextIntervalMillis(int retryCount) - { - int interval = config.getBaseIntervalMillis() * ((int) Math.pow(2.0, (double) retryCount)); - if (interval > config.getMaxIntervalMillis()) { - return config.getMaxIntervalMillis(); - } - return interval; + public void setBaseIntervalMillis(int baseIntervalMillis) { + this.baseIntervalMillis = baseIntervalMillis; } - public int getBaseIntervalMillis() - { - return config.getBaseIntervalMillis(); + public int getMaxIntervalMillis() { + return maxIntervalMillis; } - public int getMaxIntervalMillis() - { - return config.getMaxIntervalMillis(); + public void setMaxIntervalMillis(int maxIntervalMillis) { + this.maxIntervalMillis = maxIntervalMillis; } - @Override - public String toString() - { - return "ExponentialBackOffRetryStrategy{" + - "config=" + config + - "} " + super.toString(); + void validateValues() { + validate(); } - public static class Config - extends RetryStrategy.Config - implements Validatable - { - @Min(10) - private int baseIntervalMillis = 400; - @Min(10) - private int maxIntervalMillis = 30 * 1000; - - public int getBaseIntervalMillis() - { - return baseIntervalMillis; - } - - public void setBaseIntervalMillis(int baseIntervalMillis) - { - this.baseIntervalMillis = baseIntervalMillis; - } - - public int getMaxIntervalMillis() - { - return maxIntervalMillis; - } - - public void setMaxIntervalMillis(int maxIntervalMillis) - { - this.maxIntervalMillis = maxIntervalMillis; - } - - void validateValues() - { - validate(); - } - - @Override - public String toString() - { - return "Config{" + - "baseIntervalMillis=" + baseIntervalMillis + - ", maxIntervalMillis=" + maxIntervalMillis + - "} " + super.toString(); - } + @Override + public String toString() { + return "Config{" + + "baseIntervalMillis=" + + baseIntervalMillis + + ", maxIntervalMillis=" + + maxIntervalMillis + + "} " + + super.toString(); } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/RetryStrategy.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/RetryStrategy.java index dc4649a7..3a29e362 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/RetryStrategy.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/RetryStrategy.java @@ -16,59 +16,45 @@ package org.komamitsu.fluency.fluentd.ingester.sender.retry; -import org.komamitsu.fluency.validation.Validatable; import org.komamitsu.fluency.validation.annotation.Min; -public abstract class RetryStrategy -{ - private final Config config; +public abstract class RetryStrategy { + private final Config config; - protected RetryStrategy(Config config) - { - this.config = config; - } + protected RetryStrategy(Config config) { + this.config = config; + } - public abstract int getNextIntervalMillis(int retryCount); + public abstract int getNextIntervalMillis(int retryCount); - public boolean isRetriedOver(int retryCount) - { - return retryCount > config.getMaxRetryCount(); - } + public boolean isRetriedOver(int retryCount) { + return retryCount > config.getMaxRetryCount(); + } - public int getMaxRetryCount() - { - return config.getMaxRetryCount(); - } + public int getMaxRetryCount() { + return config.getMaxRetryCount(); + } - @Override - public String toString() - { - return "RetryStrategy{" + - "config=" + config + - '}'; - } + @Override + public String toString() { + return "RetryStrategy{" + "config=" + config + '}'; + } - public static class Config - { - @Min(0) - private int maxRetryCount = 7; + public static class Config { + @Min(0) + private int maxRetryCount = 7; - public int getMaxRetryCount() - { - return maxRetryCount; - } + public int getMaxRetryCount() { + return maxRetryCount; + } - public void setMaxRetryCount(int maxRetryCount) - { - this.maxRetryCount = maxRetryCount; - } + public void setMaxRetryCount(int maxRetryCount) { + this.maxRetryCount = maxRetryCount; + } - @Override - public String toString() - { - return "Config{" + - "maxRetryCount=" + maxRetryCount + - '}'; - } + @Override + public String toString() { + return "Config{" + "maxRetryCount=" + maxRetryCount + '}'; } + } } diff --git a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/recordformat/FluentdRecordFormatter.java b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/recordformat/FluentdRecordFormatter.java index 2a538a01..20f8d02e 100644 --- a/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/recordformat/FluentdRecordFormatter.java +++ b/fluency-fluentd/src/main/java/org/komamitsu/fluency/fluentd/recordformat/FluentdRecordFormatter.java @@ -17,106 +17,85 @@ package org.komamitsu.fluency.fluentd.recordformat; import com.fasterxml.jackson.core.JsonProcessingException; -import org.komamitsu.fluency.recordformat.AbstractRecordFormatter; -import org.komamitsu.fluency.recordformat.RecordFormatter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Map; +import org.komamitsu.fluency.recordformat.AbstractRecordFormatter; +import org.komamitsu.fluency.recordformat.RecordFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class FluentdRecordFormatter - extends AbstractRecordFormatter - implements RecordFormatter -{ - private static final Logger LOG = LoggerFactory.getLogger(FluentdRecordFormatter.class); +public class FluentdRecordFormatter extends AbstractRecordFormatter implements RecordFormatter { + private static final Logger LOG = LoggerFactory.getLogger(FluentdRecordFormatter.class); - public FluentdRecordFormatter() - { - this(new Config()); - } + public FluentdRecordFormatter() { + this(new Config()); + } - public FluentdRecordFormatter(Config config) - { - super(config); - } + public FluentdRecordFormatter(Config config) { + super(config); + } - @Override - public byte[] format(String tag, Object timestamp, Map data) - { - try { - return objectMapperForMessagePack.writeValueAsBytes(Arrays.asList(timestamp, data)); - } - catch (JsonProcessingException e) { - throw new IllegalArgumentException( - String.format( - "Failed to convert the record to MessagePack format: cause=%s, tag=%s, timestamp=%s, recordCount=%d", - e.getMessage(), - tag, timestamp, data.size()) - ); - } + @Override + public byte[] format(String tag, Object timestamp, Map data) { + try { + return objectMapperForMessagePack.writeValueAsBytes(Arrays.asList(timestamp, data)); + } catch (JsonProcessingException e) { + throw new IllegalArgumentException( + String.format( + "Failed to convert the record to MessagePack format: cause=%s, tag=%s, timestamp=%s, recordCount=%d", + e.getMessage(), tag, timestamp, data.size())); } + } - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, byte[] mapValue, int offset, int len) - { - try { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - // 2 items array - outputStream.write(0x92); - objectMapperForMessagePack.writeValue(outputStream, timestamp); - outputStream.write(mapValue, offset, len); - outputStream.close(); + @Override + public byte[] formatFromMessagePack( + String tag, Object timestamp, byte[] mapValue, int offset, int len) { + try { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + // 2 items array + outputStream.write(0x92); + objectMapperForMessagePack.writeValue(outputStream, timestamp); + outputStream.write(mapValue, offset, len); + outputStream.close(); - return outputStream.toByteArray(); - } - catch (IOException e) { - throw new IllegalArgumentException( - String.format( - "Failed to convert the record to MessagePack format: cause=%s, tag=%s, timestamp=%s, dataSize=%s", - e.getMessage(), - tag, timestamp, mapValue.length) - ); - } + return outputStream.toByteArray(); + } catch (IOException e) { + throw new IllegalArgumentException( + String.format( + "Failed to convert the record to MessagePack format: cause=%s, tag=%s, timestamp=%s, dataSize=%s", + e.getMessage(), tag, timestamp, mapValue.length)); } + } - @Override - public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) - { - int mapValueLen = mapValue.remaining(); - try { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - // 2 items array - outputStream.write(0x92); - objectMapperForMessagePack.writeValue(outputStream, timestamp); - while (mapValue.hasRemaining()) { - outputStream.write(mapValue.get()); - } - outputStream.close(); + @Override + public byte[] formatFromMessagePack(String tag, Object timestamp, ByteBuffer mapValue) { + int mapValueLen = mapValue.remaining(); + try { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + // 2 items array + outputStream.write(0x92); + objectMapperForMessagePack.writeValue(outputStream, timestamp); + while (mapValue.hasRemaining()) { + outputStream.write(mapValue.get()); + } + outputStream.close(); - return outputStream.toByteArray(); - } - catch (IOException e) { - throw new IllegalArgumentException( - String.format( - "Failed to convert the record to MessagePack format: cause=%s, tag=%s, timestamp=%s, dataSize=%s", - e.getMessage(), - tag, timestamp, mapValueLen) - ); - } + return outputStream.toByteArray(); + } catch (IOException e) { + throw new IllegalArgumentException( + String.format( + "Failed to convert the record to MessagePack format: cause=%s, tag=%s, timestamp=%s, dataSize=%s", + e.getMessage(), tag, timestamp, mapValueLen)); } + } - @Override - public String formatName() - { - return "fluentd-packedforward"; - } + @Override + public String formatName() { + return "fluentd-packedforward"; + } - public static class Config - extends RecordFormatter.Config - { - } + public static class Config extends RecordFormatter.Config {} } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/buffer/BufferForFluentdTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/buffer/BufferForFluentdTest.java index 04b634e6..6c0281ed 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/buffer/BufferForFluentdTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/buffer/BufferForFluentdTest.java @@ -16,8 +16,21 @@ package org.komamitsu.fluency.buffer; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; import org.komamitsu.fluency.EventTime; import org.komamitsu.fluency.fluentd.ingester.FluentdIngester; @@ -33,271 +46,296 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class BufferForFluentdTest -{ - private final FluentdRecordFormatter fluentdRecordFormatter = - new FluentdRecordFormatter(new FluentdRecordFormatter.Config()); +class BufferForFluentdTest { + private final FluentdRecordFormatter fluentdRecordFormatter = + new FluentdRecordFormatter(new FluentdRecordFormatter.Config()); - @Test - void withFluentdFormat() - throws IOException, InterruptedException - { - for (Integer loopCount : Arrays.asList(100, 1000, 10000, 100000)) { - new BufferTestHelper().baseTestMessageBuffer(loopCount, true, true, false, - new Buffer(new Buffer.Config(), fluentdRecordFormatter)); - new BufferTestHelper().baseTestMessageBuffer(loopCount, false, true, false, - new Buffer(new Buffer.Config(), fluentdRecordFormatter)); - new BufferTestHelper().baseTestMessageBuffer(loopCount, true, false, false, - new Buffer(new Buffer.Config(), fluentdRecordFormatter)); - new BufferTestHelper().baseTestMessageBuffer(loopCount, false, false, false, - new Buffer(new Buffer.Config(), fluentdRecordFormatter)); - new BufferTestHelper().baseTestMessageBuffer(loopCount, true, true, true, - new Buffer(new Buffer.Config(), fluentdRecordFormatter)); - new BufferTestHelper().baseTestMessageBuffer(loopCount, false, true, true, - new Buffer(new Buffer.Config(), fluentdRecordFormatter)); - new BufferTestHelper().baseTestMessageBuffer(loopCount, true, false, true, - new Buffer(new Buffer.Config(), fluentdRecordFormatter)); - new BufferTestHelper().baseTestMessageBuffer(loopCount, false, false, true, - new Buffer(new Buffer.Config(), fluentdRecordFormatter)); - } + @Test + void withFluentdFormat() throws IOException, InterruptedException { + for (Integer loopCount : Arrays.asList(100, 1000, 10000, 100000)) { + new BufferTestHelper() + .baseTestMessageBuffer( + loopCount, + true, + true, + false, + new Buffer(new Buffer.Config(), fluentdRecordFormatter)); + new BufferTestHelper() + .baseTestMessageBuffer( + loopCount, + false, + true, + false, + new Buffer(new Buffer.Config(), fluentdRecordFormatter)); + new BufferTestHelper() + .baseTestMessageBuffer( + loopCount, + true, + false, + false, + new Buffer(new Buffer.Config(), fluentdRecordFormatter)); + new BufferTestHelper() + .baseTestMessageBuffer( + loopCount, + false, + false, + false, + new Buffer(new Buffer.Config(), fluentdRecordFormatter)); + new BufferTestHelper() + .baseTestMessageBuffer( + loopCount, true, true, true, new Buffer(new Buffer.Config(), fluentdRecordFormatter)); + new BufferTestHelper() + .baseTestMessageBuffer( + loopCount, + false, + true, + true, + new Buffer(new Buffer.Config(), fluentdRecordFormatter)); + new BufferTestHelper() + .baseTestMessageBuffer( + loopCount, + true, + false, + true, + new Buffer(new Buffer.Config(), fluentdRecordFormatter)); + new BufferTestHelper() + .baseTestMessageBuffer( + loopCount, + false, + false, + true, + new Buffer(new Buffer.Config(), fluentdRecordFormatter)); } + } - static class BufferTestHelper - { - private static final Logger LOG = LoggerFactory.getLogger(BufferTestHelper.class); - private final String longStr; - private HashMap tagCounts; - private String minName; - private String maxName; - private int minAge; - private int maxAge; - private int longCommentCount; + static class BufferTestHelper { + private static final Logger LOG = LoggerFactory.getLogger(BufferTestHelper.class); + private final String longStr; + private HashMap tagCounts; + private String minName; + private String maxName; + private int minAge; + private int maxAge; + private int longCommentCount; - BufferTestHelper() - { - StringBuilder stringBuilder = new StringBuilder(); - for (int i = 0; i < 300; i++) { - stringBuilder.append("xxxxxxxxxx"); - } - longStr = stringBuilder.toString(); - tagCounts = new HashMap<>(); - minName = "zzzzzzzzzzzzzzzzzzzz"; - maxName = ""; - minAge = Integer.MAX_VALUE; - maxAge = Integer.MIN_VALUE; - longCommentCount = 0; - } + BufferTestHelper() { + StringBuilder stringBuilder = new StringBuilder(); + for (int i = 0; i < 300; i++) { + stringBuilder.append("xxxxxxxxxx"); + } + longStr = stringBuilder.toString(); + tagCounts = new HashMap<>(); + minName = "zzzzzzzzzzzzzzzzzzzz"; + maxName = ""; + minAge = Integer.MAX_VALUE; + maxAge = Integer.MIN_VALUE; + longCommentCount = 0; + } - void baseTestMessageBuffer(final int loopCount, final boolean multiTags, final boolean syncFlush, final boolean eventTime, final Buffer buffer) - throws IOException, InterruptedException - { - assertThat(buffer.getBufferUsage()).isEqualTo(0f); - assertThat(buffer.getAllocatedSize()).isEqualTo(0L); - assertThat(buffer.getBufferedDataSize()).isEqualTo(0L); + void baseTestMessageBuffer( + final int loopCount, + final boolean multiTags, + final boolean syncFlush, + final boolean eventTime, + final Buffer buffer) + throws IOException, InterruptedException { + assertThat(buffer.getBufferUsage()).isEqualTo(0f); + assertThat(buffer.getAllocatedSize()).isEqualTo(0L); + assertThat(buffer.getBufferedDataSize()).isEqualTo(0L); - final int concurrency = 4; - final CountDownLatch latch = new CountDownLatch(concurrency); + final int concurrency = 4; + final CountDownLatch latch = new CountDownLatch(concurrency); - TCPSender.Config config = new TCPSender.Config(); - config.setPort(24229); - final MockTCPSender sender = new MockTCPSender(config); - Ingester ingester = new FluentdIngester(new FluentdIngester.Config(), sender); + TCPSender.Config config = new TCPSender.Config(); + config.setPort(24229); + final MockTCPSender sender = new MockTCPSender(config); + Ingester ingester = new FluentdIngester(new FluentdIngester.Config(), sender); - Runnable emitTask = () -> { - try { - for (int i = 0; i < loopCount; i++) { - HashMap data = new HashMap<>(); - data.put("name", String.format("komamitsu%06d", i)); - data.put("age", i); - data.put("comment", i % 31 == 0 ? longStr : "hello"); - String tag = multiTags ? String.format("foodb%d.bartbl%d", i % 4, i % 4) : "foodb.bartbl"; - if (eventTime) { - buffer.append(tag, new EventTime((int) (System.currentTimeMillis() / 1000), 999999999), data); - } - else { - buffer.append(tag, System.currentTimeMillis(), data); - } - } - latch.countDown(); + Runnable emitTask = + () -> { + try { + for (int i = 0; i < loopCount; i++) { + HashMap data = new HashMap<>(); + data.put("name", String.format("komamitsu%06d", i)); + data.put("age", i); + data.put("comment", i % 31 == 0 ? longStr : "hello"); + String tag = + multiTags ? String.format("foodb%d.bartbl%d", i % 4, i % 4) : "foodb.bartbl"; + if (eventTime) { + buffer.append( + tag, + new EventTime((int) (System.currentTimeMillis() / 1000), 999999999), + data); + } else { + buffer.append(tag, System.currentTimeMillis(), data); } - catch (Exception e) { - e.printStackTrace(); - } - }; - - final ExecutorService flushService = Executors.newSingleThreadExecutor(); - if (!syncFlush) { - flushService.execute(() -> { - while (!flushService.isShutdown()) { - try { - TimeUnit.MILLISECONDS.sleep(100L); - buffer.flush(ingester, false); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - catch (IOException e) { - e.printStackTrace(); - return; - } - } - }); + } + latch.countDown(); + } catch (Exception e) { + e.printStackTrace(); } + }; - long start = System.currentTimeMillis(); - final ExecutorService executorService = Executors.newFixedThreadPool(concurrency); - for (int i = 0; i < concurrency; i++) { - executorService.execute(emitTask); - } - assertTrue(latch.await(10, TimeUnit.SECONDS)); - assertThat(buffer.getBufferUsage()).isGreaterThan(0f); - assertThat(buffer.getAllocatedSize()).isGreaterThan(0L); - assertThat(buffer.getBufferedDataSize()).isGreaterThan(0L); + final ExecutorService flushService = Executors.newSingleThreadExecutor(); + if (!syncFlush) { + flushService.execute( + () -> { + while (!flushService.isShutdown()) { + try { + TimeUnit.MILLISECONDS.sleep(100L); + buffer.flush(ingester, false); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (IOException e) { + e.printStackTrace(); + return; + } + } + }); + } - buffer.flush(ingester, true); - buffer.close(); - long end = System.currentTimeMillis(); + long start = System.currentTimeMillis(); + final ExecutorService executorService = Executors.newFixedThreadPool(concurrency); + for (int i = 0; i < concurrency; i++) { + executorService.execute(emitTask); + } + assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertThat(buffer.getBufferUsage()).isGreaterThan(0f); + assertThat(buffer.getAllocatedSize()).isGreaterThan(0L); + assertThat(buffer.getBufferedDataSize()).isGreaterThan(0L); - executorService.shutdown(); - executorService.awaitTermination(10, TimeUnit.SECONDS); - if (!executorService.isTerminated()) { - executorService.shutdownNow(); - } - flushService.shutdown(); - flushService.awaitTermination(10, TimeUnit.SECONDS); - if (!flushService.isTerminated()) { - flushService.shutdownNow(); - } - buffer.close(); // Just in case - assertThat(buffer.getBufferUsage()).isEqualTo(0f); - assertThat(buffer.getAllocatedSize()).isEqualTo(0L); - assertThat(buffer.getBufferedDataSize()).isEqualTo(0L); + buffer.flush(ingester, true); + buffer.close(); + long end = System.currentTimeMillis(); - int totalLoopCount = concurrency * loopCount; + executorService.shutdown(); + executorService.awaitTermination(10, TimeUnit.SECONDS); + if (!executorService.isTerminated()) { + executorService.shutdownNow(); + } + flushService.shutdown(); + flushService.awaitTermination(10, TimeUnit.SECONDS); + if (!flushService.isTerminated()) { + flushService.shutdownNow(); + } + buffer.close(); // Just in case + assertThat(buffer.getBufferUsage()).isEqualTo(0f); + assertThat(buffer.getAllocatedSize()).isEqualTo(0L); + assertThat(buffer.getBufferedDataSize()).isEqualTo(0L); - int recordCount = 0; - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - objectMapper.disable(JsonParser.Feature.AUTO_CLOSE_SOURCE); - for (MockTCPSender.Event event : sender.getEvents()) { - byte[] bytes = event.getAllBytes(); + int totalLoopCount = concurrency * loopCount; - MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(bytes); - assertEquals(3, messageUnpacker.unpackArrayHeader()); + int recordCount = 0; + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + objectMapper.disable(JsonParser.Feature.AUTO_CLOSE_SOURCE); + for (MockTCPSender.Event event : sender.getEvents()) { + byte[] bytes = event.getAllBytes(); - String tag = messageUnpacker.unpackString(); - byte[] payload = messageUnpacker.readPayload(messageUnpacker.unpackBinaryHeader()); - messageUnpacker = MessagePack.newDefaultUnpacker(payload); - while (messageUnpacker.hasNext()) { - assertEquals(2, messageUnpacker.unpackArrayHeader()); + MessageUnpacker messageUnpacker = MessagePack.newDefaultUnpacker(bytes); + assertEquals(3, messageUnpacker.unpackArrayHeader()); - ImmutableValue timestamp = messageUnpacker.unpackValue(); + String tag = messageUnpacker.unpackString(); + byte[] payload = messageUnpacker.readPayload(messageUnpacker.unpackBinaryHeader()); + messageUnpacker = MessagePack.newDefaultUnpacker(payload); + while (messageUnpacker.hasNext()) { + assertEquals(2, messageUnpacker.unpackArrayHeader()); - int size = messageUnpacker.unpackMapHeader(); - assertEquals(3, size); - Map data = new HashMap<>(); - for (int i = 0; i < size; i++) { - String key = messageUnpacker.unpackString(); - ImmutableValue value = messageUnpacker.unpackValue(); - if (value.isStringValue()) { - data.put(key, value.asStringValue().asString()); - } - else if (value.isIntegerValue()) { - data.put(key, value.asIntegerValue().asInt()); - } - } + ImmutableValue timestamp = messageUnpacker.unpackValue(); - analyzeResult(tag, timestamp, data, start, end, eventTime); - recordCount++; - } + int size = messageUnpacker.unpackMapHeader(); + assertEquals(3, size); + Map data = new HashMap<>(); + for (int i = 0; i < size; i++) { + String key = messageUnpacker.unpackString(); + ImmutableValue value = messageUnpacker.unpackValue(); + if (value.isStringValue()) { + data.put(key, value.asStringValue().asString()); + } else if (value.isIntegerValue()) { + data.put(key, value.asIntegerValue().asInt()); } + } - assertEquals(totalLoopCount, recordCount); - - if (multiTags) { - assertEquals(4, tagCounts.size()); - for (int i = 0; i < 4; i++) { - int count = tagCounts.get(String.format("foodb%d.bartbl%d", i, i)); - assertTrue(totalLoopCount / 4 - 4 <= count && count <= totalLoopCount / 4 + 4); - } - } - else { - assertEquals(1, tagCounts.size()); - int count = tagCounts.get("foodb.bartbl"); - assertEquals(totalLoopCount, count); - } + analyzeResult(tag, timestamp, data, start, end, eventTime); + recordCount++; + } + } - assertEquals("komamitsu000000", minName); - assertEquals(String.format("komamitsu%06d", loopCount - 1), maxName); - assertEquals(0, minAge); - assertEquals(loopCount - 1, maxAge); + assertEquals(totalLoopCount, recordCount); - assertTrue(totalLoopCount / 31 - 5 <= longCommentCount && longCommentCount <= totalLoopCount / 31 + 5); + if (multiTags) { + assertEquals(4, tagCounts.size()); + for (int i = 0; i < 4; i++) { + int count = tagCounts.get(String.format("foodb%d.bartbl%d", i, i)); + assertTrue(totalLoopCount / 4 - 4 <= count && count <= totalLoopCount / 4 + 4); } + } else { + assertEquals(1, tagCounts.size()); + int count = tagCounts.get("foodb.bartbl"); + assertEquals(totalLoopCount, count); + } - private void analyzeResult(String tag, ImmutableValue timestamp, Map data, long start, long end, boolean eventTime) - { - Integer count = tagCounts.get(tag); - if (count == null) { - count = 0; - } - tagCounts.put(tag, count + 1); + assertEquals("komamitsu000000", minName); + assertEquals(String.format("komamitsu%06d", loopCount - 1), maxName); + assertEquals(0, minAge); + assertEquals(loopCount - 1, maxAge); - if (eventTime) { - assertThat(timestamp.isExtensionValue()).isTrue(); - ExtensionValue tsInEventTime = timestamp.asExtensionValue(); - assertThat(tsInEventTime.getType()).isEqualTo((byte) 0x00); - ByteBuffer secondsAndNanoSeconds = ByteBuffer.wrap(tsInEventTime.getData()); - int seconds = secondsAndNanoSeconds.getInt(); - int nanoSeconds = secondsAndNanoSeconds.getInt(); - assertTrue(start / 1000 <= seconds && seconds <= end / 1000); - assertThat(nanoSeconds).isEqualTo(999999999); - } - else { - assertThat(timestamp.isIntegerValue()).isTrue(); - long tsInEpochMilli = timestamp.asIntegerValue().asLong(); - assertTrue(start <= tsInEpochMilli && tsInEpochMilli <= end); - } + assertTrue( + totalLoopCount / 31 - 5 <= longCommentCount + && longCommentCount <= totalLoopCount / 31 + 5); + } - assertEquals(3, data.size()); - String name = (String) data.get("name"); - int age = (Integer) data.get("age"); - String comment = (String) data.get("comment"); - if (name.compareTo(minName) < 0) { - minName = name; - } - if (name.compareTo(maxName) > 0) { - maxName = name; - } - if (age < minAge) { - minAge = age; - } - if (age > maxAge) { - maxAge = age; - } + private void analyzeResult( + String tag, + ImmutableValue timestamp, + Map data, + long start, + long end, + boolean eventTime) { + Integer count = tagCounts.get(tag); + if (count == null) { + count = 0; + } + tagCounts.put(tag, count + 1); - if (comment.equals("hello")) { - // expected - } - else if (comment.equals(longStr)) { - longCommentCount++; - } - else { - assertTrue(false); - } - } + if (eventTime) { + assertThat(timestamp.isExtensionValue()).isTrue(); + ExtensionValue tsInEventTime = timestamp.asExtensionValue(); + assertThat(tsInEventTime.getType()).isEqualTo((byte) 0x00); + ByteBuffer secondsAndNanoSeconds = ByteBuffer.wrap(tsInEventTime.getData()); + int seconds = secondsAndNanoSeconds.getInt(); + int nanoSeconds = secondsAndNanoSeconds.getInt(); + assertTrue(start / 1000 <= seconds && seconds <= end / 1000); + assertThat(nanoSeconds).isEqualTo(999999999); + } else { + assertThat(timestamp.isIntegerValue()).isTrue(); + long tsInEpochMilli = timestamp.asIntegerValue().asLong(); + assertTrue(start <= tsInEpochMilli && tsInEpochMilli <= end); + } + + assertEquals(3, data.size()); + String name = (String) data.get("name"); + int age = (Integer) data.get("age"); + String comment = (String) data.get("comment"); + if (name.compareTo(minName) < 0) { + minName = name; + } + if (name.compareTo(maxName) > 0) { + maxName = name; + } + if (age < minAge) { + minAge = age; + } + if (age > maxAge) { + maxAge = age; + } + + if (comment.equals("hello")) { + // expected + } else if (comment.equals(longStr)) { + longCommentCount++; + } else { + assertTrue(false); + } } -} \ No newline at end of file + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/AbstractFluentdServer.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/AbstractFluentdServer.java index bb319b4b..44327b9a 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/AbstractFluentdServer.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/AbstractFluentdServer.java @@ -16,19 +16,7 @@ package org.komamitsu.fluency.fluentd; -import org.msgpack.core.MessagePack; -import org.msgpack.core.MessageUnpacker; -import org.msgpack.value.ExtensionValue; -import org.msgpack.value.ImmutableArrayValue; -import org.msgpack.value.ImmutableValue; -import org.msgpack.value.MapValue; -import org.msgpack.value.RawValue; -import org.msgpack.value.StringValue; -import org.msgpack.value.Value; -import org.msgpack.value.ValueFactory; -import org.msgpack.value.ValueType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.io.PipedInputStream; @@ -42,243 +30,244 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import org.msgpack.core.MessagePack; +import org.msgpack.core.MessageUnpacker; +import org.msgpack.value.ExtensionValue; +import org.msgpack.value.ImmutableArrayValue; +import org.msgpack.value.ImmutableValue; +import org.msgpack.value.MapValue; +import org.msgpack.value.StringValue; +import org.msgpack.value.Value; +import org.msgpack.value.ValueFactory; +import org.msgpack.value.ValueType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; +public abstract class AbstractFluentdServer extends MockTCPServer { + private static final Logger LOG = LoggerFactory.getLogger(AbstractFluentdServer.class); + private static final Charset CHARSET = StandardCharsets.UTF_8; + private FluentdEventHandler fluentdEventHandler; -public abstract class AbstractFluentdServer - extends MockTCPServer -{ - private static final Logger LOG = LoggerFactory.getLogger(AbstractFluentdServer.class); - private static final Charset CHARSET = StandardCharsets.UTF_8; - private FluentdEventHandler fluentdEventHandler; + AbstractFluentdServer(boolean sslEnabled) { + super(sslEnabled); + } - AbstractFluentdServer(boolean sslEnabled) - { - super(sslEnabled); + @Override + protected synchronized MockTCPServer.EventHandler getEventHandler() { + if (this.fluentdEventHandler == null) { + this.fluentdEventHandler = new FluentdEventHandler(getFluentdEventHandler()); } + return fluentdEventHandler; + } - @Override - protected synchronized MockTCPServer.EventHandler getEventHandler() - { - if (this.fluentdEventHandler == null) { - this.fluentdEventHandler = new FluentdEventHandler(getFluentdEventHandler()); - } - return fluentdEventHandler; - } + protected abstract EventHandler getFluentdEventHandler(); - protected abstract EventHandler getFluentdEventHandler(); + public interface EventHandler { + void onConnect(Socket acceptSocket); - public interface EventHandler - { - void onConnect(Socket acceptSocket); + void onReceive(String tag, long timestampMillis, MapValue data); - void onReceive(String tag, long timestampMillis, MapValue data); + void onClose(Socket accpetSocket); + } - void onClose(Socket accpetSocket); - } + private static class FluentdEventHandler implements MockTCPServer.EventHandler { + private static final StringValue KEY_OPTION_SIZE = ValueFactory.newString("size"); + private static final StringValue KEY_OPTION_CHUNK = ValueFactory.newString("chunk"); + private final EventHandler eventHandler; + private final ExecutorService executorService = Executors.newCachedThreadPool(); + private final Map fluentdTasks = + new ConcurrentHashMap(); - private static class FluentdEventHandler - implements MockTCPServer.EventHandler - { - private static final StringValue KEY_OPTION_SIZE = ValueFactory.newString("size"); - private static final StringValue KEY_OPTION_CHUNK = ValueFactory.newString("chunk"); - private final EventHandler eventHandler; - private final ExecutorService executorService = Executors.newCachedThreadPool(); - private final Map fluentdTasks = new ConcurrentHashMap(); + private FluentdEventHandler(EventHandler eventHandler) { + this.eventHandler = eventHandler; + } - private FluentdEventHandler(EventHandler eventHandler) - { - this.eventHandler = eventHandler; - } + private void ack(Socket acceptSocket, String ackResponseToken) throws IOException { + byte[] ackResponseTokenBytes = ackResponseToken.getBytes(CHARSET); + ByteBuffer byteBuffer = + ByteBuffer.allocate( + 1 /* map header */ + + 1 /* key header */ + + 3 /* key body */ + + 2 /* value header(including len) */ + + ackResponseTokenBytes.length); - private void ack(Socket acceptSocket, String ackResponseToken) - throws IOException - { - byte[] ackResponseTokenBytes = ackResponseToken.getBytes(CHARSET); - ByteBuffer byteBuffer = ByteBuffer.allocate( - 1 /* map header */ + - 1 /* key header */ + - 3 /* key body */ + - 2 /* value header(including len) */ + - ackResponseTokenBytes.length); + byteBuffer.put((byte) 0x81); /* map header */ + byteBuffer.put((byte) 0xA3); /* key header */ + byteBuffer.put("ack".getBytes(CHARSET)); /* key body */ + byteBuffer.put((byte) 0xD9); + byteBuffer.put((byte) ackResponseTokenBytes.length); + byteBuffer.put(ackResponseTokenBytes); + byteBuffer.flip(); + acceptSocket.getOutputStream().write(byteBuffer.array()); + } - byteBuffer.put((byte) 0x81); /* map header */ - byteBuffer.put((byte) 0xA3); /* key header */ - byteBuffer.put("ack".getBytes(CHARSET)); /* key body */ - byteBuffer.put((byte) 0xD9); - byteBuffer.put((byte) ackResponseTokenBytes.length); - byteBuffer.put(ackResponseTokenBytes); - byteBuffer.flip(); - acceptSocket.getOutputStream().write(byteBuffer.array()); - } + @Override + public void onConnect(final Socket acceptSocket) { + eventHandler.onConnect(acceptSocket); + try { + FluentdAcceptTask fluentdAcceptTask = new FluentdAcceptTask(acceptSocket); + fluentdTasks.put(acceptSocket, fluentdAcceptTask); + executorService.execute(fluentdAcceptTask); + } catch (IOException e) { + fluentdTasks.remove(acceptSocket); + throw new IllegalStateException("Failed to create FluentdAcceptTask", e); + } + } - @Override - public void onConnect(final Socket acceptSocket) - { - eventHandler.onConnect(acceptSocket); - try { - FluentdAcceptTask fluentdAcceptTask = new FluentdAcceptTask(acceptSocket); - fluentdTasks.put(acceptSocket, fluentdAcceptTask); - executorService.execute(fluentdAcceptTask); - } - catch (IOException e) { - fluentdTasks.remove(acceptSocket); - throw new IllegalStateException("Failed to create FluentdAcceptTask", e); - } - } + @Override + public void onReceive(Socket acceptSocket, int len, byte[] data) { + FluentdAcceptTask fluentdAcceptTask = fluentdTasks.get(acceptSocket); + if (fluentdAcceptTask == null) { + throw new IllegalStateException("fluentAccept is null: this=" + this); + } - @Override - public void onReceive(Socket acceptSocket, int len, byte[] data) - { - FluentdAcceptTask fluentdAcceptTask = fluentdTasks.get(acceptSocket); - if (fluentdAcceptTask == null) { - throw new IllegalStateException("fluentAccept is null: this=" + this); - } + LOG.trace( + "onReceived: local.port={}, remote.port={}, dataLen={}", + acceptSocket.getLocalPort(), + acceptSocket.getPort(), + len); + try { + fluentdAcceptTask.getPipedOutputStream().write(data, 0, len); + fluentdAcceptTask.getPipedOutputStream().flush(); + } catch (IOException e) { + throw new RuntimeException("Failed to call PipedOutputStream.write(): this=" + this); + } + } - LOG.trace("onReceived: local.port={}, remote.port={}, dataLen={}", acceptSocket.getLocalPort(), acceptSocket.getPort(), len); - try { - fluentdAcceptTask.getPipedOutputStream().write(data, 0, len); - fluentdAcceptTask.getPipedOutputStream().flush(); - } - catch (IOException e) { - throw new RuntimeException("Failed to call PipedOutputStream.write(): this=" + this); - } + @Override + public void onClose(Socket acceptSocket) { + eventHandler.onClose(acceptSocket); + FluentdAcceptTask fluentdAcceptTask = fluentdTasks.remove(acceptSocket); + if (fluentdAcceptTask == null) { + return; + } + try { + if (fluentdAcceptTask.getPipedInputStream() != null) { + fluentdAcceptTask.getPipedInputStream().close(); } - - @Override - public void onClose(Socket acceptSocket) - { - eventHandler.onClose(acceptSocket); - FluentdAcceptTask fluentdAcceptTask = fluentdTasks.remove(acceptSocket); - if (fluentdAcceptTask == null) { - return; - } - try { - if (fluentdAcceptTask.getPipedInputStream() != null) { - fluentdAcceptTask.getPipedInputStream().close(); - } - } - catch (IOException e) { - LOG.warn("Failed to close PipedInputStream"); - } - try { - if (fluentdAcceptTask.getPipedOutputStream() != null) { - fluentdAcceptTask.getPipedOutputStream().close(); - } - } - catch (IOException e) { - LOG.warn("Failed to close PipedOutputStream"); - } + } catch (IOException e) { + LOG.warn("Failed to close PipedInputStream"); + } + try { + if (fluentdAcceptTask.getPipedOutputStream() != null) { + fluentdAcceptTask.getPipedOutputStream().close(); } + } catch (IOException e) { + LOG.warn("Failed to close PipedOutputStream"); + } + } - private class FluentdAcceptTask - implements Runnable - { - private final Socket acceptSocket; - private final PipedInputStream pipedInputStream; - private final PipedOutputStream pipedOutputStream; + private class FluentdAcceptTask implements Runnable { + private final Socket acceptSocket; + private final PipedInputStream pipedInputStream; + private final PipedOutputStream pipedOutputStream; - private FluentdAcceptTask(Socket acceptSocket) - throws IOException - { - this.acceptSocket = acceptSocket; - this.pipedOutputStream = new PipedOutputStream(); - this.pipedInputStream = new PipedInputStream(pipedOutputStream); - } + private FluentdAcceptTask(Socket acceptSocket) throws IOException { + this.acceptSocket = acceptSocket; + this.pipedOutputStream = new PipedOutputStream(); + this.pipedInputStream = new PipedInputStream(pipedOutputStream); + } - PipedInputStream getPipedInputStream() - { - return pipedInputStream; - } + PipedInputStream getPipedInputStream() { + return pipedInputStream; + } - PipedOutputStream getPipedOutputStream() - { - return pipedOutputStream; - } + PipedOutputStream getPipedOutputStream() { + return pipedOutputStream; + } - @Override - public void run() - { - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(pipedInputStream); + @Override + public void run() { + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(pipedInputStream); - try { - while (!executorService.isTerminated()) { - ImmutableValue value; - try { - if (!unpacker.hasNext()) { - break; - } - value = unpacker.unpackValue(); - LOG.trace("Received a value: local.port={}, remote.port={}", acceptSocket.getLocalPort(), acceptSocket.getPort()); - } - catch (Exception e) { - LOG.debug("Fluentd accept task received IOException: {}", e.getMessage()); - break; - } - assertEquals(ValueType.ARRAY, value.getValueType()); - ImmutableArrayValue rootValue = value.asArrayValue(); - assertEquals(rootValue.size(), 3); - - String tag = rootValue.get(0).toString(); - Value secondValue = rootValue.get(1); - - // PackedForward - byte[] packedBytes = secondValue.asRawValue().asByteArray(); - MessageUnpacker eventsUnpacker = MessagePack.newDefaultUnpacker(packedBytes); - while (eventsUnpacker.hasNext()) { - ImmutableArrayValue arrayValue = eventsUnpacker.unpackValue().asArrayValue(); - assertEquals(2, arrayValue.size()); - Value timestampValue = arrayValue.get(0); - MapValue mapValue = arrayValue.get(1).asMapValue(); - long timestampMillis; - if (timestampValue.isIntegerValue()) { - timestampMillis = timestampValue.asIntegerValue().asLong() * 1000; - } - else if (timestampValue.isExtensionValue()) { - ExtensionValue extensionValue = timestampValue.asExtensionValue(); - if (extensionValue.getType() != 0) { - throw new IllegalArgumentException("Unexpected extension type: " + extensionValue.getType()); - } - byte[] data = extensionValue.getData(); - long seconds = ByteBuffer.wrap(data, 0, 4).order(ByteOrder.BIG_ENDIAN).getInt(); - long nanos = ByteBuffer.wrap(data, 4, 4).order(ByteOrder.BIG_ENDIAN).getInt(); - timestampMillis = seconds * 1000 + nanos / 1000000; - } - else { - throw new IllegalArgumentException("Unexpected value type: " + timestampValue); - } - eventHandler.onReceive(tag, timestampMillis, mapValue); - } + try { + while (!executorService.isTerminated()) { + ImmutableValue value; + try { + if (!unpacker.hasNext()) { + break; + } + value = unpacker.unpackValue(); + LOG.trace( + "Received a value: local.port={}, remote.port={}", + acceptSocket.getLocalPort(), + acceptSocket.getPort()); + } catch (Exception e) { + LOG.debug("Fluentd accept task received IOException: {}", e.getMessage()); + break; + } + assertEquals(ValueType.ARRAY, value.getValueType()); + ImmutableArrayValue rootValue = value.asArrayValue(); + assertEquals(rootValue.size(), 3); - // Option - Map map = rootValue.get(2).asMapValue().map(); - // "size" - assertEquals(map.get(KEY_OPTION_SIZE).asIntegerValue().asLong(), packedBytes.length); - // "chunk" - Value chunk = map.get(KEY_OPTION_CHUNK); - if (chunk != null) { - ack(acceptSocket, chunk.asStringValue().asString()); - } - } + String tag = rootValue.get(0).toString(); + Value secondValue = rootValue.get(1); - try { - LOG.debug("Closing unpacker: this={}, local.port={}, remote.port={}", this, acceptSocket.getLocalPort(), acceptSocket.getPort()); - unpacker.close(); - } - catch (IOException e) { - LOG.warn("Failed to close unpacker quietly: this={}, unpacker={}", this, unpacker); - } - } - catch (Throwable e) { - LOG.error("Fluentd server failed: this=" + this + ", local.port=" + acceptSocket.getLocalPort() + ", remote.port=" + acceptSocket.getPort(), e); - try { - acceptSocket.close(); - } - catch (IOException e1) { - LOG.warn("Failed to close accept socket quietly", e1); - } + // PackedForward + byte[] packedBytes = secondValue.asRawValue().asByteArray(); + MessageUnpacker eventsUnpacker = MessagePack.newDefaultUnpacker(packedBytes); + while (eventsUnpacker.hasNext()) { + ImmutableArrayValue arrayValue = eventsUnpacker.unpackValue().asArrayValue(); + assertEquals(2, arrayValue.size()); + Value timestampValue = arrayValue.get(0); + MapValue mapValue = arrayValue.get(1).asMapValue(); + long timestampMillis; + if (timestampValue.isIntegerValue()) { + timestampMillis = timestampValue.asIntegerValue().asLong() * 1000; + } else if (timestampValue.isExtensionValue()) { + ExtensionValue extensionValue = timestampValue.asExtensionValue(); + if (extensionValue.getType() != 0) { + throw new IllegalArgumentException( + "Unexpected extension type: " + extensionValue.getType()); } + byte[] data = extensionValue.getData(); + long seconds = ByteBuffer.wrap(data, 0, 4).order(ByteOrder.BIG_ENDIAN).getInt(); + long nanos = ByteBuffer.wrap(data, 4, 4).order(ByteOrder.BIG_ENDIAN).getInt(); + timestampMillis = seconds * 1000 + nanos / 1000000; + } else { + throw new IllegalArgumentException("Unexpected value type: " + timestampValue); + } + eventHandler.onReceive(tag, timestampMillis, mapValue); + } + + // Option + Map map = rootValue.get(2).asMapValue().map(); + // "size" + assertEquals(map.get(KEY_OPTION_SIZE).asIntegerValue().asLong(), packedBytes.length); + // "chunk" + Value chunk = map.get(KEY_OPTION_CHUNK); + if (chunk != null) { + ack(acceptSocket, chunk.asStringValue().asString()); } + } + + try { + LOG.debug( + "Closing unpacker: this={}, local.port={}, remote.port={}", + this, + acceptSocket.getLocalPort(), + acceptSocket.getPort()); + unpacker.close(); + } catch (IOException e) { + LOG.warn("Failed to close unpacker quietly: this={}, unpacker={}", this, unpacker); + } + } catch (Throwable e) { + LOG.error( + "Fluentd server failed: this=" + + this + + ", local.port=" + + acceptSocket.getLocalPort() + + ", remote.port=" + + acceptSocket.getPort(), + e); + try { + acceptSocket.close(); + } catch (IOException e1) { + LOG.warn("Failed to close accept socket quietly", e1); + } } + } } + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ConfigurableTestServer.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ConfigurableTestServer.java index 3e16625a..b74826c7 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ConfigurableTestServer.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ConfigurableTestServer.java @@ -25,87 +25,76 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; -public class ConfigurableTestServer -{ - private final boolean sslEnabled; +public class ConfigurableTestServer { + private final boolean sslEnabled; - public ConfigurableTestServer(boolean sslEnabled) - { - this.sslEnabled = sslEnabled; - } + public ConfigurableTestServer(boolean sslEnabled) { + this.sslEnabled = sslEnabled; + } - Exception run(final WithClientSocket withClientSocket, final WithServerPort withServerPort, long timeoutMilli) - throws Throwable - { - ExecutorService executorService = Executors.newCachedThreadPool(); - final AtomicReference serverSocket = new AtomicReference(); + Exception run( + final WithClientSocket withClientSocket, + final WithServerPort withServerPort, + long timeoutMilli) + throws Throwable { + ExecutorService executorService = Executors.newCachedThreadPool(); + final AtomicReference serverSocket = new AtomicReference(); - try { - if (sslEnabled) { - serverSocket.set(SSLTestSocketFactories.createServerSocket()); - } - else { - serverSocket.set(new ServerSocket(0)); - } + try { + if (sslEnabled) { + serverSocket.set(SSLTestSocketFactories.createServerSocket()); + } else { + serverSocket.set(new ServerSocket(0)); + } - final int serverPort = serverSocket.get().getLocalPort(); + final int serverPort = serverSocket.get().getLocalPort(); - Future serverSideFuture = executorService.submit(new Callable() - { + Future serverSideFuture = + executorService.submit( + new Callable() { @Override - public Void call() - throws Exception - { - Socket acceptSocket = serverSocket.get().accept(); - try { - withClientSocket.run(acceptSocket); - } - finally { - acceptSocket.close(); - } - return null; + public Void call() throws Exception { + Socket acceptSocket = serverSocket.get().accept(); + try { + withClientSocket.run(acceptSocket); + } finally { + acceptSocket.close(); + } + return null; } - }); + }); - Future testTaskFuture = executorService.submit(new Callable() - { + Future testTaskFuture = + executorService.submit( + new Callable() { @Override - public Void call() - throws Exception - { - withServerPort.run(serverPort); - return null; + public Void call() throws Exception { + withServerPort.run(serverPort); + return null; } - }); + }); - try { - testTaskFuture.get(timeoutMilli, TimeUnit.MILLISECONDS); - } - catch (Exception e) { - return e; - } - finally { - serverSideFuture.get(); - } - } - finally { - executorService.shutdownNow(); - if (serverSocket.get() != null) { - serverSocket.get().close(); - } - } - return null; + try { + testTaskFuture.get(timeoutMilli, TimeUnit.MILLISECONDS); + } catch (Exception e) { + return e; + } finally { + serverSideFuture.get(); + } + } finally { + executorService.shutdownNow(); + if (serverSocket.get() != null) { + serverSocket.get().close(); + } } + return null; + } - interface WithClientSocket - { - void run(Socket clientSocket) - throws Exception; - } + interface WithClientSocket { + void run(Socket clientSocket) throws Exception; + } - interface WithServerPort - { - void run(int serverPort) - throws Exception; - } + interface WithServerPort { + void run(int serverPort) throws Exception; + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/EventTimeTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/EventTimeTest.java index 63bd4d9c..7b4da294 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/EventTimeTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/EventTimeTest.java @@ -16,99 +16,96 @@ package org.komamitsu.fluency.fluentd; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.ByteBuffer; import org.junit.jupiter.api.Test; import org.komamitsu.fluency.EventTime; import org.msgpack.jackson.dataformat.MessagePackFactory; -import java.nio.ByteBuffer; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertThrows; - -class EventTimeTest -{ - @Test - void instantiation() +class EventTimeTest { + @Test + void instantiation() { { - { - long now = System.currentTimeMillis(); - EventTime eventTime = EventTime.fromEpoch(now / 1000); - assertThat(eventTime.getSeconds()).isEqualTo(now / 1000); - assertThat(eventTime.getNanoseconds()).isEqualTo(0L); - } - - { - long now = System.currentTimeMillis(); - EventTime eventTime = EventTime.fromEpoch(now / 1000, 999999999L); - assertThat(eventTime.getSeconds()).isEqualTo(now / 1000); - assertThat(eventTime.getNanoseconds()).isEqualTo(999999999L); - } - - { - long now = System.currentTimeMillis(); - EventTime eventTime = EventTime.fromEpochMilli(now); - assertThat(eventTime.getSeconds()).isEqualTo(now / 1000); - assertThat(eventTime.getNanoseconds()).isEqualTo(now % 1000 * 1000000); - } + long now = System.currentTimeMillis(); + EventTime eventTime = EventTime.fromEpoch(now / 1000); + assertThat(eventTime.getSeconds()).isEqualTo(now / 1000); + assertThat(eventTime.getNanoseconds()).isEqualTo(0L); + } - { - EventTime eventTime = EventTime.fromEpoch(0xFFFFFFFFL, 0xFFFFFFFFL); - assertThat(eventTime.getSeconds()).isEqualTo(0xFFFFFFFFL); - assertThat(eventTime.getNanoseconds()).isEqualTo(0xFFFFFFFFL); - } + { + long now = System.currentTimeMillis(); + EventTime eventTime = EventTime.fromEpoch(now / 1000, 999999999L); + assertThat(eventTime.getSeconds()).isEqualTo(now / 1000); + assertThat(eventTime.getNanoseconds()).isEqualTo(999999999L); } - @Test - void instantiationWithTooLargeSeconds() { - assertThrows(IllegalArgumentException.class, () -> { - EventTime.fromEpoch(0x100000000L, 0L); - }); + long now = System.currentTimeMillis(); + EventTime eventTime = EventTime.fromEpochMilli(now); + assertThat(eventTime.getSeconds()).isEqualTo(now / 1000); + assertThat(eventTime.getNanoseconds()).isEqualTo(now % 1000 * 1000000); } - @Test - void instantiationWithTooLargeNanoSeconds() { - assertThrows(IllegalArgumentException.class, () -> { - EventTime.fromEpoch(0L, 0x100000000L); - }); + EventTime eventTime = EventTime.fromEpoch(0xFFFFFFFFL, 0xFFFFFFFFL); + assertThat(eventTime.getSeconds()).isEqualTo(0xFFFFFFFFL); + assertThat(eventTime.getNanoseconds()).isEqualTo(0xFFFFFFFFL); } + } + + @Test + void instantiationWithTooLargeSeconds() { + assertThrows( + IllegalArgumentException.class, + () -> { + EventTime.fromEpoch(0x100000000L, 0L); + }); + } + + @Test + void instantiationWithTooLargeNanoSeconds() { + assertThrows( + IllegalArgumentException.class, + () -> { + EventTime.fromEpoch(0L, 0x100000000L); + }); + } - @Test - void serialize() - throws JsonProcessingException + @Test + void serialize() throws JsonProcessingException { { - { - long now = System.currentTimeMillis(); - EventTime eventTime = EventTime.fromEpoch(now / 1000, 999999999); - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - byte[] bytes = objectMapper.writeValueAsBytes(eventTime); - ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xD7); - assertThat(byteBuffer.get()).isEqualTo((byte) 0x00); - assertThat(byteBuffer.getInt()).isEqualTo((int) (now / 1000)); - assertThat(byteBuffer.getInt()).isEqualTo(999999999); - } + long now = System.currentTimeMillis(); + EventTime eventTime = EventTime.fromEpoch(now / 1000, 999999999); + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + byte[] bytes = objectMapper.writeValueAsBytes(eventTime); + ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); + assertThat(byteBuffer.get()).isEqualTo((byte) 0xD7); + assertThat(byteBuffer.get()).isEqualTo((byte) 0x00); + assertThat(byteBuffer.getInt()).isEqualTo((int) (now / 1000)); + assertThat(byteBuffer.getInt()).isEqualTo(999999999); + } - { - EventTime eventTime = EventTime.fromEpoch(0xFFEEDDCCL, 0xFEDCBA98L); - ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - byte[] bytes = objectMapper.writeValueAsBytes(eventTime); - ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xD7); - assertThat(byteBuffer.get()).isEqualTo((byte) 0x00); + { + EventTime eventTime = EventTime.fromEpoch(0xFFEEDDCCL, 0xFEDCBA98L); + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + byte[] bytes = objectMapper.writeValueAsBytes(eventTime); + ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); + assertThat(byteBuffer.get()).isEqualTo((byte) 0xD7); + assertThat(byteBuffer.get()).isEqualTo((byte) 0x00); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xFF); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xEE); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xDD); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xCC); + assertThat(byteBuffer.get()).isEqualTo((byte) 0xFF); + assertThat(byteBuffer.get()).isEqualTo((byte) 0xEE); + assertThat(byteBuffer.get()).isEqualTo((byte) 0xDD); + assertThat(byteBuffer.get()).isEqualTo((byte) 0xCC); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xFE); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xDC); - assertThat(byteBuffer.get()).isEqualTo((byte) 0xBA); - assertThat(byteBuffer.get()).isEqualTo((byte) 0x98); - } + assertThat(byteBuffer.get()).isEqualTo((byte) 0xFE); + assertThat(byteBuffer.get()).isEqualTo((byte) 0xDC); + assertThat(byteBuffer.get()).isEqualTo((byte) 0xBA); + assertThat(byteBuffer.get()).isEqualTo((byte) 0x98); } + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyBuilderForFluentdTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyBuilderForFluentdTest.java index dd461baa..08d009f5 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyBuilderForFluentdTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyBuilderForFluentdTest.java @@ -16,7 +16,18 @@ package org.komamitsu.fluency.fluentd; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.*; + import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.channels.SocketChannel; +import java.util.Arrays; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; import org.junit.jupiter.api.Test; import org.komamitsu.fluency.Fluency; import org.komamitsu.fluency.buffer.Buffer; @@ -29,382 +40,361 @@ import org.komamitsu.fluency.fluentd.recordformat.FluentdRecordFormatter; import org.komamitsu.fluency.flusher.Flusher; -import javax.net.ssl.SSLSocket; -import javax.net.ssl.SSLSocketFactory; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.channels.SocketChannel; -import java.util.Arrays; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.Mockito.*; - -class FluencyBuilderForFluentdTest -{ - private void assertBuffer(Buffer buffer) - { - assertThat(buffer.getMaxBufferSize()).isEqualTo(512 * 1024 * 1024L); - assertThat(buffer.getFileBackupDir()).isNull(); - assertThat(buffer.bufferFormatType()).isEqualTo("packed_forward"); - assertThat(buffer.getChunkExpandRatio()).isEqualTo(2f); - assertThat(buffer.getChunkRetentionSize()).isEqualTo(4 * 1024 * 1024); - assertThat(buffer.getChunkInitialSize()).isEqualTo(1 * 1024 * 1024); - assertThat(buffer.getChunkRetentionTimeMillis()).isEqualTo(1000); - assertThat(buffer.getJvmHeapBufferMode()).isFalse(); - } - - private void assertFlusher(Flusher flusher) - { - assertThat(flusher.isTerminated()).isFalse(); - assertThat(flusher.getFlushAttemptIntervalMillis()).isEqualTo(600); - assertThat(flusher.getWaitUntilBufferFlushed()).isEqualTo(60); - assertThat(flusher.getWaitUntilTerminated()).isEqualTo(60); +class FluencyBuilderForFluentdTest { + private void assertBuffer(Buffer buffer) { + assertThat(buffer.getMaxBufferSize()).isEqualTo(512 * 1024 * 1024L); + assertThat(buffer.getFileBackupDir()).isNull(); + assertThat(buffer.bufferFormatType()).isEqualTo("packed_forward"); + assertThat(buffer.getChunkExpandRatio()).isEqualTo(2f); + assertThat(buffer.getChunkRetentionSize()).isEqualTo(4 * 1024 * 1024); + assertThat(buffer.getChunkInitialSize()).isEqualTo(1 * 1024 * 1024); + assertThat(buffer.getChunkRetentionTimeMillis()).isEqualTo(1000); + assertThat(buffer.getJvmHeapBufferMode()).isFalse(); + } + + private void assertFlusher(Flusher flusher) { + assertThat(flusher.isTerminated()).isFalse(); + assertThat(flusher.getFlushAttemptIntervalMillis()).isEqualTo(600); + assertThat(flusher.getWaitUntilBufferFlushed()).isEqualTo(60); + assertThat(flusher.getWaitUntilTerminated()).isEqualTo(60); + } + + private void assertDefaultRetryableSender( + RetryableSender sender, Class expectedBaseClass) { + assertThat(sender.getRetryStrategy()).isInstanceOf(ExponentialBackOffRetryStrategy.class); + ExponentialBackOffRetryStrategy retryStrategy = + (ExponentialBackOffRetryStrategy) sender.getRetryStrategy(); + assertThat(retryStrategy.getMaxRetryCount()).isEqualTo(7); + assertThat(retryStrategy.getBaseIntervalMillis()).isEqualTo(400); + assertThat(sender.getBaseSender()).isInstanceOf(expectedBaseClass); + } + + private void assertDefaultFluentdSender( + FluentdSender sender, + String expectedHost, + int expectedPort, + Class expectedBaseClass) { + assertThat(sender).isInstanceOf(RetryableSender.class); + RetryableSender retryableSender = (RetryableSender) sender; + assertDefaultRetryableSender(retryableSender, expectedBaseClass); + + assertThat(retryableSender.getBaseSender()).isInstanceOf(InetSocketSender.class); + InetSocketSender networkSender = + (InetSocketSender) retryableSender.getBaseSender(); + assertThat(networkSender.getHost()).isEqualTo(expectedHost); + assertThat(networkSender.getPort()).isEqualTo(expectedPort); + assertThat(networkSender.getConnectionTimeoutMilli()).isEqualTo(5000); + assertThat(networkSender.getReadTimeoutMilli()).isEqualTo(5000); + + FailureDetector failureDetector = networkSender.getFailureDetector(); + assertThat(failureDetector).isNull(); + } + + @Test + void build() throws IOException { + try (Fluency fluency = new FluencyBuilderForFluentd().build()) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (FluentdSender) fluency.getFlusher().getIngester().getSender(), + "127.0.0.1", + 24224, + TCPSender.class); } - - private void assertDefaultRetryableSender(RetryableSender sender, Class expectedBaseClass) - { - assertThat(sender.getRetryStrategy()).isInstanceOf(ExponentialBackOffRetryStrategy.class); - ExponentialBackOffRetryStrategy retryStrategy = (ExponentialBackOffRetryStrategy) sender.getRetryStrategy(); - assertThat(retryStrategy.getMaxRetryCount()).isEqualTo(7); - assertThat(retryStrategy.getBaseIntervalMillis()).isEqualTo(400); - assertThat(sender.getBaseSender()).isInstanceOf(expectedBaseClass); + } + + @Test + void buildWithCustomPort() throws IOException { + try (Fluency fluency = new FluencyBuilderForFluentd().build(54321)) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (FluentdSender) fluency.getFlusher().getIngester().getSender(), + "127.0.0.1", + 54321, + TCPSender.class); } - - private void assertDefaultFluentdSender(FluentdSender sender, String expectedHost, int expectedPort, Class expectedBaseClass) - { - assertThat(sender).isInstanceOf(RetryableSender.class); - RetryableSender retryableSender = (RetryableSender) sender; - assertDefaultRetryableSender(retryableSender, expectedBaseClass); - - assertThat(retryableSender.getBaseSender()).isInstanceOf(InetSocketSender.class); - InetSocketSender networkSender = (InetSocketSender) retryableSender.getBaseSender(); - assertThat(networkSender.getHost()).isEqualTo(expectedHost); - assertThat(networkSender.getPort()).isEqualTo(expectedPort); - assertThat(networkSender.getConnectionTimeoutMilli()).isEqualTo(5000); - assertThat(networkSender.getReadTimeoutMilli()).isEqualTo(5000); - - FailureDetector failureDetector = networkSender.getFailureDetector(); - assertThat(failureDetector).isNull(); + } + + @Test + void buildWithCustomHostAndPort() throws IOException { + try (Fluency fluency = new FluencyBuilderForFluentd().build("192.168.0.99", 54321)) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (FluentdSender) fluency.getFlusher().getIngester().getSender(), + "192.168.0.99", + 54321, + TCPSender.class); } - - @Test - void build() - throws IOException - { - try (Fluency fluency = new FluencyBuilderForFluentd().build()) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (FluentdSender) fluency.getFlusher().getIngester().getSender(), - "127.0.0.1", - 24224, - TCPSender.class); - } + } + + @Test + void buildWithSsl() throws IOException { + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(true); + try (Fluency fluency = builder.build()) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (FluentdSender) fluency.getFlusher().getIngester().getSender(), + "127.0.0.1", + 24224, + SSLSender.class); } - - @Test - void buildWithCustomPort() - throws IOException - { - try (Fluency fluency = new FluencyBuilderForFluentd().build(54321)) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (FluentdSender) fluency.getFlusher().getIngester().getSender(), - "127.0.0.1", - 54321, - TCPSender.class); - } + } + + @Test + void buildWithSslAndCustomPort() throws IOException { + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(true); + try (Fluency fluency = builder.build(54321)) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (FluentdSender) fluency.getFlusher().getIngester().getSender(), + "127.0.0.1", + 54321, + SSLSender.class); } - - @Test - void buildWithCustomHostAndPort() - throws IOException - { - try (Fluency fluency = new FluencyBuilderForFluentd().build("192.168.0.99", 54321)) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (FluentdSender) fluency.getFlusher().getIngester().getSender(), - "192.168.0.99", - 54321, - TCPSender.class); - } + } + + @Test + void buildWithSslAndCustomHostAndPort() throws IOException { + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(true); + try (Fluency fluency = builder.build("192.168.0.99", 54321)) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (FluentdSender) fluency.getFlusher().getIngester().getSender(), + "192.168.0.99", + 54321, + SSLSender.class); } - - @Test - void buildWithSsl() - throws IOException - { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(true); - try (Fluency fluency = builder.build()) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (FluentdSender) fluency.getFlusher().getIngester().getSender(), - "127.0.0.1", - 24224, - SSLSender.class); - } + } + + @Test + void buildWithSslSocketFactory() throws IOException { + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(true); + SSLSocketFactory sslSocketFactory = mock(SSLSocketFactory.class); + SSLSocket socket = mock(SSLSocket.class); + RuntimeException expectedException = new RuntimeException("Expected"); + doThrow(expectedException).when(socket).connect(any(InetSocketAddress.class), anyInt()); + doNothing().when(socket).close(); + doReturn(socket).when(sslSocketFactory).createSocket(); + builder.setSslSocketFactory(sslSocketFactory); + builder.setSenderMaxRetryCount(0); + try (Fluency fluency = builder.build("192.168.0.99", 54321)) { + fluency.emit("mytag", ImmutableMap.of("hello", "world")); } - - @Test - void buildWithSslAndCustomPort() - throws IOException - { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(true); - try (Fluency fluency = builder.build(54321)) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (FluentdSender) fluency.getFlusher().getIngester().getSender(), - "127.0.0.1", - 54321, - SSLSender.class); + verify(socket).connect(any(InetSocketAddress.class), anyInt()); + } + + @Test + void buildWithComplexConfig() throws IOException { + String tmpdir = System.getProperty("java.io.tmpdir"); + assertThat(tmpdir).isNotNull(); + + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setFlushAttemptIntervalMillis(200); + builder.setMaxBufferSize(Long.MAX_VALUE); + builder.setBufferChunkInitialSize(7 * 1024 * 1024); + builder.setBufferChunkRetentionSize(13 * 1024 * 1024); + builder.setBufferChunkRetentionTimeMillis(19 * 1000); + builder.setJvmHeapBufferMode(true); + builder.setSenderMaxRetryCount(99); + builder.setSenderBaseRetryIntervalMillis(20); + builder.setSenderMaxRetryIntervalMillis(100000); + builder.setConnectionTimeoutMilli(12345); + builder.setReadTimeoutMilli(9876); + builder.setAckResponseMode(true); + builder.setWaitUntilBufferFlushed(42); + builder.setWaitUntilFlusherTerminated(24); + builder.setFileBackupDir(tmpdir); + + try (Fluency fluency = + builder.build( + Arrays.asList( + new InetSocketAddress("333.333.333.333", 11111), + new InetSocketAddress("444.444.444.444", 22222)))) { + + assertThat(fluency.getBuffer()).isInstanceOf(Buffer.class); + Buffer buffer = fluency.getBuffer(); + assertThat(buffer.getMaxBufferSize()).isEqualTo(Long.MAX_VALUE); + assertThat(buffer.getFileBackupDir()).isEqualTo(tmpdir); + assertThat(buffer.bufferFormatType()).isEqualTo("packed_forward"); + assertThat(buffer.getChunkRetentionTimeMillis()).isEqualTo(19 * 1000); + assertThat(buffer.getChunkExpandRatio()).isEqualTo(2f); + assertThat(buffer.getChunkInitialSize()).isEqualTo(7 * 1024 * 1024); + assertThat(buffer.getChunkRetentionSize()).isEqualTo(13 * 1024 * 1024); + assertThat(buffer.getJvmHeapBufferMode()).isEqualTo(true); + + Flusher flusher = fluency.getFlusher(); + assertThat(flusher.isTerminated()).isFalse(); + assertThat(flusher.getFlushAttemptIntervalMillis()).isEqualTo(200); + assertThat(flusher.getWaitUntilBufferFlushed()).isEqualTo(42); + assertThat(flusher.getWaitUntilTerminated()).isEqualTo(24); + + assertThat(flusher.getIngester().getSender()).isInstanceOf(RetryableSender.class); + RetryableSender retryableSender = (RetryableSender) flusher.getIngester().getSender(); + assertThat(retryableSender.getRetryStrategy()) + .isInstanceOf(ExponentialBackOffRetryStrategy.class); + ExponentialBackOffRetryStrategy retryStrategy = + (ExponentialBackOffRetryStrategy) retryableSender.getRetryStrategy(); + assertThat(retryStrategy.getMaxRetryCount()).isEqualTo(99); + assertThat(retryStrategy.getBaseIntervalMillis()).isEqualTo(20); + assertThat(retryStrategy.getMaxIntervalMillis()).isEqualTo(100000); + + assertThat(retryableSender.getBaseSender()).isInstanceOf(MultiSender.class); + MultiSender multiSender = (MultiSender) retryableSender.getBaseSender(); + assertThat(multiSender.getSenders().size()).isEqualTo(2); + + assertThat(multiSender.getSenders().get(0)).isInstanceOf(TCPSender.class); + { + TCPSender sender = (TCPSender) multiSender.getSenders().get(0); + assertThat(sender.getHost()).isEqualTo("333.333.333.333"); + assertThat(sender.getPort()).isEqualTo(11111); + assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(12345); + assertThat(sender.getReadTimeoutMilli()).isEqualTo(9876); + + FailureDetector failureDetector = sender.getFailureDetector(); + assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); + assertThat(failureDetector.getFailureDetectStrategy()) + .isInstanceOf(PhiAccrualFailureDetectStrategy.class); + assertThat(failureDetector.getHeartbeater()).isInstanceOf(TCPHeartbeater.class); + { + TCPHeartbeater hb = (TCPHeartbeater) failureDetector.getHeartbeater(); + assertThat(hb.getHost()).isEqualTo("333.333.333.333"); + assertThat(hb.getPort()).isEqualTo(11111); } - } - - @Test - void buildWithSslAndCustomHostAndPort() - throws IOException - { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(true); - try (Fluency fluency = builder.build("192.168.0.99", 54321)) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (FluentdSender) fluency.getFlusher().getIngester().getSender(), - "192.168.0.99", - 54321, - SSLSender.class); + assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); + } + + assertThat(multiSender.getSenders().get(1)).isInstanceOf(TCPSender.class); + { + TCPSender sender = (TCPSender) multiSender.getSenders().get(1); + assertThat(sender.getHost()).isEqualTo("444.444.444.444"); + assertThat(sender.getPort()).isEqualTo(22222); + assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(12345); + assertThat(sender.getReadTimeoutMilli()).isEqualTo(9876); + + FailureDetector failureDetector = sender.getFailureDetector(); + assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); + assertThat(failureDetector.getFailureDetectStrategy()) + .isInstanceOf(PhiAccrualFailureDetectStrategy.class); + assertThat(failureDetector.getHeartbeater()).isInstanceOf(TCPHeartbeater.class); + { + TCPHeartbeater hb = (TCPHeartbeater) failureDetector.getHeartbeater(); + assertThat(hb.getHost()).isEqualTo("444.444.444.444"); + assertThat(hb.getPort()).isEqualTo(22222); } + assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); + } } - - @Test - void buildWithSslSocketFactory() - throws IOException - { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(true); - SSLSocketFactory sslSocketFactory = mock(SSLSocketFactory.class); - SSLSocket socket = mock(SSLSocket.class); - RuntimeException expectedException = new RuntimeException("Expected"); - doThrow(expectedException).when(socket).connect(any(InetSocketAddress.class), anyInt()); - doNothing().when(socket).close(); - doReturn(socket).when(sslSocketFactory).createSocket(); - builder.setSslSocketFactory(sslSocketFactory); - builder.setSenderMaxRetryCount(0); - try (Fluency fluency = builder.build("192.168.0.99", 54321)) { - fluency.emit("mytag", ImmutableMap.of("hello", "world")); - } - verify(socket).connect(any(InetSocketAddress.class), anyInt()); - } - - @Test - void buildWithComplexConfig() - throws IOException - { - String tmpdir = System.getProperty("java.io.tmpdir"); - assertThat(tmpdir).isNotNull(); - - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setFlushAttemptIntervalMillis(200); - builder.setMaxBufferSize(Long.MAX_VALUE); - builder.setBufferChunkInitialSize(7 * 1024 * 1024); - builder.setBufferChunkRetentionSize(13 * 1024 * 1024); - builder.setBufferChunkRetentionTimeMillis(19 * 1000); - builder.setJvmHeapBufferMode(true); - builder.setSenderMaxRetryCount(99); - builder.setSenderBaseRetryIntervalMillis(20); - builder.setSenderMaxRetryIntervalMillis(100000); - builder.setConnectionTimeoutMilli(12345); - builder.setReadTimeoutMilli(9876); - builder.setAckResponseMode(true); - builder.setWaitUntilBufferFlushed(42); - builder.setWaitUntilFlusherTerminated(24); - builder.setFileBackupDir(tmpdir); - - try (Fluency fluency = builder.build( - Arrays.asList( - new InetSocketAddress("333.333.333.333", 11111), - new InetSocketAddress("444.444.444.444", 22222)))) { - - assertThat(fluency.getBuffer()).isInstanceOf(Buffer.class); - Buffer buffer = fluency.getBuffer(); - assertThat(buffer.getMaxBufferSize()).isEqualTo(Long.MAX_VALUE); - assertThat(buffer.getFileBackupDir()).isEqualTo(tmpdir); - assertThat(buffer.bufferFormatType()).isEqualTo("packed_forward"); - assertThat(buffer.getChunkRetentionTimeMillis()).isEqualTo(19 * 1000); - assertThat(buffer.getChunkExpandRatio()).isEqualTo(2f); - assertThat(buffer.getChunkInitialSize()).isEqualTo(7 * 1024 * 1024); - assertThat(buffer.getChunkRetentionSize()).isEqualTo(13 * 1024 * 1024); - assertThat(buffer.getJvmHeapBufferMode()).isEqualTo(true); - - Flusher flusher = fluency.getFlusher(); - assertThat(flusher.isTerminated()).isFalse(); - assertThat(flusher.getFlushAttemptIntervalMillis()).isEqualTo(200); - assertThat(flusher.getWaitUntilBufferFlushed()).isEqualTo(42); - assertThat(flusher.getWaitUntilTerminated()).isEqualTo(24); - - assertThat(flusher.getIngester().getSender()).isInstanceOf(RetryableSender.class); - RetryableSender retryableSender = (RetryableSender) flusher.getIngester().getSender(); - assertThat(retryableSender.getRetryStrategy()).isInstanceOf(ExponentialBackOffRetryStrategy.class); - ExponentialBackOffRetryStrategy retryStrategy = (ExponentialBackOffRetryStrategy) retryableSender.getRetryStrategy(); - assertThat(retryStrategy.getMaxRetryCount()).isEqualTo(99); - assertThat(retryStrategy.getBaseIntervalMillis()).isEqualTo(20); - assertThat(retryStrategy.getMaxIntervalMillis()).isEqualTo(100000); - - assertThat(retryableSender.getBaseSender()).isInstanceOf(MultiSender.class); - MultiSender multiSender = (MultiSender) retryableSender.getBaseSender(); - assertThat(multiSender.getSenders().size()).isEqualTo(2); - - assertThat(multiSender.getSenders().get(0)).isInstanceOf(TCPSender.class); - { - TCPSender sender = (TCPSender) multiSender.getSenders().get(0); - assertThat(sender.getHost()).isEqualTo("333.333.333.333"); - assertThat(sender.getPort()).isEqualTo(11111); - assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(12345); - assertThat(sender.getReadTimeoutMilli()).isEqualTo(9876); - - FailureDetector failureDetector = sender.getFailureDetector(); - assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); - assertThat(failureDetector.getFailureDetectStrategy()).isInstanceOf(PhiAccrualFailureDetectStrategy.class); - assertThat(failureDetector.getHeartbeater()).isInstanceOf(TCPHeartbeater.class); - { - TCPHeartbeater hb = (TCPHeartbeater) failureDetector.getHeartbeater(); - assertThat(hb.getHost()).isEqualTo("333.333.333.333"); - assertThat(hb.getPort()).isEqualTo(11111); - } - assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); - } - - assertThat(multiSender.getSenders().get(1)).isInstanceOf(TCPSender.class); - { - TCPSender sender = (TCPSender) multiSender.getSenders().get(1); - assertThat(sender.getHost()).isEqualTo("444.444.444.444"); - assertThat(sender.getPort()).isEqualTo(22222); - assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(12345); - assertThat(sender.getReadTimeoutMilli()).isEqualTo(9876); - - - FailureDetector failureDetector = sender.getFailureDetector(); - assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); - assertThat(failureDetector.getFailureDetectStrategy()).isInstanceOf(PhiAccrualFailureDetectStrategy.class); - assertThat(failureDetector.getHeartbeater()).isInstanceOf(TCPHeartbeater.class); - { - TCPHeartbeater hb = (TCPHeartbeater) failureDetector.getHeartbeater(); - assertThat(hb.getHost()).isEqualTo("444.444.444.444"); - assertThat(hb.getPort()).isEqualTo(22222); - } - assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); - } + } + + @Test + void buildWithSslAndComplexConfig() throws IOException { + String tmpdir = System.getProperty("java.io.tmpdir"); + assertThat(tmpdir).isNotNull(); + + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(true); + builder.setFlushAttemptIntervalMillis(200); + builder.setMaxBufferSize(Long.MAX_VALUE); + builder.setBufferChunkInitialSize(7 * 1024 * 1024); + builder.setBufferChunkRetentionSize(13 * 1024 * 1024); + builder.setJvmHeapBufferMode(true); + builder.setSenderMaxRetryCount(99); + builder.setSenderBaseRetryIntervalMillis(20); + builder.setSenderMaxRetryIntervalMillis(100000); + builder.setConnectionTimeoutMilli(12345); + builder.setReadTimeoutMilli(9876); + builder.setAckResponseMode(true); + builder.setWaitUntilBufferFlushed(42); + builder.setWaitUntilFlusherTerminated(24); + builder.setFileBackupDir(tmpdir); + + try (Fluency fluency = + builder.build( + Arrays.asList( + new InetSocketAddress("333.333.333.333", 11111), + new InetSocketAddress("444.444.444.444", 22222)))) { + + assertThat(fluency.getFlusher().getIngester().getSender()) + .isInstanceOf(RetryableSender.class); + RetryableSender retryableSender = + (RetryableSender) fluency.getFlusher().getIngester().getSender(); + assertThat(retryableSender.getRetryStrategy()) + .isInstanceOf(ExponentialBackOffRetryStrategy.class); + ExponentialBackOffRetryStrategy retryStrategy = + (ExponentialBackOffRetryStrategy) retryableSender.getRetryStrategy(); + assertThat(retryStrategy.getMaxRetryCount()).isEqualTo(99); + assertThat(retryStrategy.getBaseIntervalMillis()).isEqualTo(20); + assertThat(retryStrategy.getMaxIntervalMillis()).isEqualTo(100000); + + assertThat(retryableSender.getBaseSender()).isInstanceOf(MultiSender.class); + MultiSender multiSender = (MultiSender) retryableSender.getBaseSender(); + assertThat(multiSender.getSenders().size()).isEqualTo(2); + + assertThat(multiSender.getSenders().get(0)).isInstanceOf(SSLSender.class); + { + SSLSender sender = (SSLSender) multiSender.getSenders().get(0); + assertThat(sender.getHost()).isEqualTo("333.333.333.333"); + assertThat(sender.getPort()).isEqualTo(11111); + assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(12345); + assertThat(sender.getReadTimeoutMilli()).isEqualTo(9876); + + FailureDetector failureDetector = sender.getFailureDetector(); + assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); + assertThat(failureDetector.getFailureDetectStrategy()) + .isInstanceOf(PhiAccrualFailureDetectStrategy.class); + assertThat(failureDetector.getHeartbeater()).isInstanceOf(SSLHeartbeater.class); + { + SSLHeartbeater hb = (SSLHeartbeater) failureDetector.getHeartbeater(); + assertThat(hb.getHost()).isEqualTo("333.333.333.333"); + assertThat(hb.getPort()).isEqualTo(11111); } - } - - @Test - void buildWithSslAndComplexConfig() - throws IOException - { - String tmpdir = System.getProperty("java.io.tmpdir"); - assertThat(tmpdir).isNotNull(); - - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(true); - builder.setFlushAttemptIntervalMillis(200); - builder.setMaxBufferSize(Long.MAX_VALUE); - builder.setBufferChunkInitialSize(7 * 1024 * 1024); - builder.setBufferChunkRetentionSize(13 * 1024 * 1024); - builder.setJvmHeapBufferMode(true); - builder.setSenderMaxRetryCount(99); - builder.setSenderBaseRetryIntervalMillis(20); - builder.setSenderMaxRetryIntervalMillis(100000); - builder.setConnectionTimeoutMilli(12345); - builder.setReadTimeoutMilli(9876); - builder.setAckResponseMode(true); - builder.setWaitUntilBufferFlushed(42); - builder.setWaitUntilFlusherTerminated(24); - builder.setFileBackupDir(tmpdir); - - try (Fluency fluency = builder.build( - Arrays.asList( - new InetSocketAddress("333.333.333.333", 11111), - new InetSocketAddress("444.444.444.444", 22222)))) { - - assertThat(fluency.getFlusher().getIngester().getSender()).isInstanceOf(RetryableSender.class); - RetryableSender retryableSender = (RetryableSender) fluency.getFlusher().getIngester().getSender(); - assertThat(retryableSender.getRetryStrategy()).isInstanceOf(ExponentialBackOffRetryStrategy.class); - ExponentialBackOffRetryStrategy retryStrategy = (ExponentialBackOffRetryStrategy) retryableSender.getRetryStrategy(); - assertThat(retryStrategy.getMaxRetryCount()).isEqualTo(99); - assertThat(retryStrategy.getBaseIntervalMillis()).isEqualTo(20); - assertThat(retryStrategy.getMaxIntervalMillis()).isEqualTo(100000); - - assertThat(retryableSender.getBaseSender()).isInstanceOf(MultiSender.class); - MultiSender multiSender = (MultiSender) retryableSender.getBaseSender(); - assertThat(multiSender.getSenders().size()).isEqualTo(2); - - assertThat(multiSender.getSenders().get(0)).isInstanceOf(SSLSender.class); - { - SSLSender sender = (SSLSender) multiSender.getSenders().get(0); - assertThat(sender.getHost()).isEqualTo("333.333.333.333"); - assertThat(sender.getPort()).isEqualTo(11111); - assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(12345); - assertThat(sender.getReadTimeoutMilli()).isEqualTo(9876); - - FailureDetector failureDetector = sender.getFailureDetector(); - assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); - assertThat(failureDetector.getFailureDetectStrategy()).isInstanceOf(PhiAccrualFailureDetectStrategy.class); - assertThat(failureDetector.getHeartbeater()).isInstanceOf(SSLHeartbeater.class); - { - SSLHeartbeater hb = (SSLHeartbeater) failureDetector.getHeartbeater(); - assertThat(hb.getHost()).isEqualTo("333.333.333.333"); - assertThat(hb.getPort()).isEqualTo(11111); - } - assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); - } - - assertThat(multiSender.getSenders().get(1)).isInstanceOf(SSLSender.class); - { - SSLSender sender = (SSLSender) multiSender.getSenders().get(1); - assertThat(sender.getHost()).isEqualTo("444.444.444.444"); - assertThat(sender.getPort()).isEqualTo(22222); - assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(12345); - assertThat(sender.getReadTimeoutMilli()).isEqualTo(9876); - - FailureDetector failureDetector = sender.getFailureDetector(); - assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); - assertThat(failureDetector.getFailureDetectStrategy()).isInstanceOf(PhiAccrualFailureDetectStrategy.class); - assertThat(failureDetector.getHeartbeater()).isInstanceOf(SSLHeartbeater.class); - { - SSLHeartbeater hb = (SSLHeartbeater) failureDetector.getHeartbeater(); - assertThat(hb.getHost()).isEqualTo("444.444.444.444"); - assertThat(hb.getPort()).isEqualTo(22222); - } - assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); - } + assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); + } + + assertThat(multiSender.getSenders().get(1)).isInstanceOf(SSLSender.class); + { + SSLSender sender = (SSLSender) multiSender.getSenders().get(1); + assertThat(sender.getHost()).isEqualTo("444.444.444.444"); + assertThat(sender.getPort()).isEqualTo(22222); + assertThat(sender.getConnectionTimeoutMilli()).isEqualTo(12345); + assertThat(sender.getReadTimeoutMilli()).isEqualTo(9876); + + FailureDetector failureDetector = sender.getFailureDetector(); + assertThat(failureDetector.getFailureIntervalMillis()).isEqualTo(3 * 1000); + assertThat(failureDetector.getFailureDetectStrategy()) + .isInstanceOf(PhiAccrualFailureDetectStrategy.class); + assertThat(failureDetector.getHeartbeater()).isInstanceOf(SSLHeartbeater.class); + { + SSLHeartbeater hb = (SSLHeartbeater) failureDetector.getHeartbeater(); + assertThat(hb.getHost()).isEqualTo("444.444.444.444"); + assertThat(hb.getPort()).isEqualTo(22222); } + assertThat(failureDetector.getHeartbeater().getIntervalMillis()).isEqualTo(1000); + } } - - @Test - void defaultRecordFormatter() - { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - assertThat(builder.getRecordFormatter()).isInstanceOf(FluentdRecordFormatter.class); - } - - @Test - void customRecordFormatter() - { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setRecordFormatter(new CustomFluentdRecordFormatter()); - assertThat(builder.getRecordFormatter()).isInstanceOf(CustomFluentdRecordFormatter.class); - } - - private static class CustomFluentdRecordFormatter extends FluentdRecordFormatter - { - } + } + + @Test + void defaultRecordFormatter() { + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + assertThat(builder.getRecordFormatter()).isInstanceOf(FluentdRecordFormatter.class); + } + + @Test + void customRecordFormatter() { + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setRecordFormatter(new CustomFluentdRecordFormatter()); + assertThat(builder.getRecordFormatter()).isInstanceOf(CustomFluentdRecordFormatter.class); + } + + private static class CustomFluentdRecordFormatter extends FluentdRecordFormatter {} } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyTest.java index ec799abb..f5852dc7 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyTest.java @@ -16,10 +16,28 @@ package org.komamitsu.fluency.fluentd; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; +import static org.mockito.Mockito.mock; + import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -39,259 +57,250 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; -import static org.mockito.Mockito.mock; - -class FluencyTest -{ - private static final Logger LOG = LoggerFactory.getLogger(FluencyTest.class); - private static final StringValue KEY_OPTION_SIZE = ValueFactory.newString("size"); - private static final StringValue KEY_OPTION_CHUNK = ValueFactory.newString("chunk"); - private Ingester ingester; - - @BeforeEach - void setUp() - { - ingester = mock(Ingester.class); - } - - @Test - void testSenderErrorHandler() - throws IOException, InterruptedException - { - final CountDownLatch countDownLatch = new CountDownLatch(1); - final AtomicReference errorContainer = new AtomicReference<>(); - - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSenderMaxRetryCount(1); - builder.setErrorHandler(e -> { - errorContainer.set(e); - countDownLatch.countDown(); +class FluencyTest { + private static final Logger LOG = LoggerFactory.getLogger(FluencyTest.class); + private static final StringValue KEY_OPTION_SIZE = ValueFactory.newString("size"); + private static final StringValue KEY_OPTION_CHUNK = ValueFactory.newString("chunk"); + private Ingester ingester; + + @BeforeEach + void setUp() { + ingester = mock(Ingester.class); + } + + @Test + void testSenderErrorHandler() throws IOException, InterruptedException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + final AtomicReference errorContainer = new AtomicReference<>(); + + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSenderMaxRetryCount(1); + builder.setErrorHandler( + e -> { + errorContainer.set(e); + countDownLatch.countDown(); }); - try (Fluency fluency = builder.build(Integer.MAX_VALUE)) { - HashMap event = new HashMap<>(); - event.put("name", "foo"); - fluency.emit("tag", event); - - if (!countDownLatch.await(10, TimeUnit.SECONDS)) { - throw new AssertionError("Timeout"); - } + try (Fluency fluency = builder.build(Integer.MAX_VALUE)) { + HashMap event = new HashMap<>(); + event.put("name", "foo"); + fluency.emit("tag", event); - assertThat(errorContainer.get()).isInstanceOf(RetryableSender.RetryOverException.class); - } - } + if (!countDownLatch.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("Timeout"); + } - static Stream sslFlagsProvider() - { - return Stream.of(false, true); + assertThat(errorContainer.get()).isInstanceOf(RetryableSender.RetryOverException.class); } - - @ParameterizedTest - @MethodSource("sslFlagsProvider") - void testWithoutAckResponse(final boolean sslEnabled) - throws Throwable - { - Exception exception = new ConfigurableTestServer(sslEnabled).run( + } + + static Stream sslFlagsProvider() { + return Stream.of(false, true); + } + + @ParameterizedTest + @MethodSource("sslFlagsProvider") + void testWithoutAckResponse(final boolean sslEnabled) throws Throwable { + Exception exception = + new ConfigurableTestServer(sslEnabled) + .run( clientSocket -> { - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream()); - assertEquals(3, unpacker.unpackArrayHeader()); - assertEquals("foo.bar", unpacker.unpackString()); - ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue(); - Map map = unpacker.unpackValue().asMapValue().map(); - assertEquals(1, map.size()); - assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt()); - unpacker.close(); + MessageUnpacker unpacker = + MessagePack.newDefaultUnpacker(clientSocket.getInputStream()); + assertEquals(3, unpacker.unpackArrayHeader()); + assertEquals("foo.bar", unpacker.unpackString()); + ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue(); + Map map = unpacker.unpackValue().asMapValue().map(); + assertEquals(1, map.size()); + assertEquals( + rawValue.asByteArray().length, + map.get(KEY_OPTION_SIZE).asIntegerValue().asInt()); + unpacker.close(); }, serverPort -> { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(sslEnabled); - if (sslEnabled) { - builder.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - } - - try (Fluency fluency = builder.build(serverPort)) { - fluency.emit("foo.bar", new HashMap<>()); - } - }, 5000); - assertNull(exception); - } - - @ParameterizedTest - @MethodSource("sslFlagsProvider") - void testWithAckResponseButNotReceiveToken(final boolean sslEnabled) - throws Throwable - { - Exception exception = new ConfigurableTestServer(sslEnabled).run( + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(sslEnabled); + if (sslEnabled) { + builder.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + } + + try (Fluency fluency = builder.build(serverPort)) { + fluency.emit("foo.bar", new HashMap<>()); + } + }, + 5000); + assertNull(exception); + } + + @ParameterizedTest + @MethodSource("sslFlagsProvider") + void testWithAckResponseButNotReceiveToken(final boolean sslEnabled) throws Throwable { + Exception exception = + new ConfigurableTestServer(sslEnabled) + .run( clientSocket -> { - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream()); - assertEquals(3, unpacker.unpackArrayHeader()); - assertEquals("foo.bar", unpacker.unpackString()); - ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue(); - Map map = unpacker.unpackValue().asMapValue().map(); - assertEquals(2, map.size()); - assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt()); - assertNotNull(map.get(KEY_OPTION_CHUNK).asRawValue().asString()); - unpacker.close(); + MessageUnpacker unpacker = + MessagePack.newDefaultUnpacker(clientSocket.getInputStream()); + assertEquals(3, unpacker.unpackArrayHeader()); + assertEquals("foo.bar", unpacker.unpackString()); + ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue(); + Map map = unpacker.unpackValue().asMapValue().map(); + assertEquals(2, map.size()); + assertEquals( + rawValue.asByteArray().length, + map.get(KEY_OPTION_SIZE).asIntegerValue().asInt()); + assertNotNull(map.get(KEY_OPTION_CHUNK).asRawValue().asString()); + unpacker.close(); }, serverPort -> { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(sslEnabled); - builder.setAckResponseMode(true); - if (sslEnabled) { - builder.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - } - - try (Fluency fluency = builder.build(serverPort)) { - fluency.emit("foo.bar", new HashMap<>()); - } - }, 5000); - assertEquals(exception.getClass(), TimeoutException.class); - } - - @ParameterizedTest - @MethodSource("sslFlagsProvider") - void testWithAckResponseButWrongReceiveToken(final boolean sslEnabled) - throws Throwable - { - Exception exception = new ConfigurableTestServer(sslEnabled).run( + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(sslEnabled); + builder.setAckResponseMode(true); + if (sslEnabled) { + builder.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + } + + try (Fluency fluency = builder.build(serverPort)) { + fluency.emit("foo.bar", new HashMap<>()); + } + }, + 5000); + assertEquals(exception.getClass(), TimeoutException.class); + } + + @ParameterizedTest + @MethodSource("sslFlagsProvider") + void testWithAckResponseButWrongReceiveToken(final boolean sslEnabled) throws Throwable { + Exception exception = + new ConfigurableTestServer(sslEnabled) + .run( clientSocket -> { - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream()); - assertEquals(3, unpacker.unpackArrayHeader()); - assertEquals("foo.bar", unpacker.unpackString()); - ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue(); - Map map = unpacker.unpackValue().asMapValue().map(); - assertEquals(2, map.size()); - assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt()); - assertNotNull(map.get(KEY_OPTION_CHUNK).asRawValue().asString()); - - MessagePacker packer = MessagePack.newDefaultPacker(clientSocket.getOutputStream()); - packer.packMapHeader(1) - .packString("ack").packString(UUID.randomUUID().toString()) - .close(); - - // Close the input stream after closing the output stream to avoid closing a socket too early - unpacker.close(); + MessageUnpacker unpacker = + MessagePack.newDefaultUnpacker(clientSocket.getInputStream()); + assertEquals(3, unpacker.unpackArrayHeader()); + assertEquals("foo.bar", unpacker.unpackString()); + ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue(); + Map map = unpacker.unpackValue().asMapValue().map(); + assertEquals(2, map.size()); + assertEquals( + rawValue.asByteArray().length, + map.get(KEY_OPTION_SIZE).asIntegerValue().asInt()); + assertNotNull(map.get(KEY_OPTION_CHUNK).asRawValue().asString()); + + MessagePacker packer = + MessagePack.newDefaultPacker(clientSocket.getOutputStream()); + packer + .packMapHeader(1) + .packString("ack") + .packString(UUID.randomUUID().toString()) + .close(); + + // Close the input stream after closing the output stream to avoid closing a + // socket too early + unpacker.close(); }, serverPort -> { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(sslEnabled); - builder.setAckResponseMode(true); - if (sslEnabled) { - builder.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - } - - try (Fluency fluency = builder.build(serverPort)) { - fluency.emit("foo.bar", new HashMap<>()); - } - }, 5000); - assertEquals(exception.getClass(), TimeoutException.class); - } - - @ParameterizedTest - @MethodSource("sslFlagsProvider") - public void testWithAckResponseWithProperToken(final boolean sslEnabled) - throws Throwable - { - Exception exception = new ConfigurableTestServer(sslEnabled).run( + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(sslEnabled); + builder.setAckResponseMode(true); + if (sslEnabled) { + builder.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + } + + try (Fluency fluency = builder.build(serverPort)) { + fluency.emit("foo.bar", new HashMap<>()); + } + }, + 5000); + assertEquals(exception.getClass(), TimeoutException.class); + } + + @ParameterizedTest + @MethodSource("sslFlagsProvider") + public void testWithAckResponseWithProperToken(final boolean sslEnabled) throws Throwable { + Exception exception = + new ConfigurableTestServer(sslEnabled) + .run( clientSocket -> { - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(clientSocket.getInputStream()); - assertEquals(3, unpacker.unpackArrayHeader()); - assertEquals("foo.bar", unpacker.unpackString()); - ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue(); - Map map = unpacker.unpackValue().asMapValue().map(); - assertEquals(2, map.size()); - assertEquals(rawValue.asByteArray().length, map.get(KEY_OPTION_SIZE).asIntegerValue().asInt()); - String ackResponseToken = map.get(KEY_OPTION_CHUNK).asRawValue().asString(); - assertNotNull(ackResponseToken); - - MessagePacker packer = MessagePack.newDefaultPacker(clientSocket.getOutputStream()); - packer.packMapHeader(1) - .packString("ack").packString(ackResponseToken) - .close(); - - // Close the input stream after closing the output stream to avoid closing a socket too early - unpacker.close(); + MessageUnpacker unpacker = + MessagePack.newDefaultUnpacker(clientSocket.getInputStream()); + assertEquals(3, unpacker.unpackArrayHeader()); + assertEquals("foo.bar", unpacker.unpackString()); + ImmutableRawValue rawValue = unpacker.unpackValue().asRawValue(); + Map map = unpacker.unpackValue().asMapValue().map(); + assertEquals(2, map.size()); + assertEquals( + rawValue.asByteArray().length, + map.get(KEY_OPTION_SIZE).asIntegerValue().asInt()); + String ackResponseToken = map.get(KEY_OPTION_CHUNK).asRawValue().asString(); + assertNotNull(ackResponseToken); + + MessagePacker packer = + MessagePack.newDefaultPacker(clientSocket.getOutputStream()); + packer.packMapHeader(1).packString("ack").packString(ackResponseToken).close(); + + // Close the input stream after closing the output stream to avoid closing a + // socket too early + unpacker.close(); }, serverPort -> { - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(sslEnabled); - builder.setAckResponseMode(true); - if (sslEnabled) { - builder.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - } - - try (Fluency fluency = builder.build(serverPort)) { - fluency.emit("foo.bar", new HashMap<>()); - } - }, 5000); - assertNull(exception); - } - - @ParameterizedTest - @MethodSource("sslFlagsProvider") - void testBufferWithJacksonModule() - throws IOException - { - AtomicBoolean serialized = new AtomicBoolean(); - SimpleModule simpleModule = new SimpleModule(); - simpleModule.addSerializer(Foo.class, new FooSerializer(serialized)); - - FluentdRecordFormatter.Config recordFormatterConfig = new FluentdRecordFormatter.Config(); - recordFormatterConfig.setJacksonModules(Collections.singletonList(simpleModule)); - - Fluency fluency = new FluencyBuilder() - .buildFromIngester(new FluentdRecordFormatter(recordFormatterConfig), ingester); - - Map event = new HashMap<>(); - Foo foo = new Foo(); - foo.s = "Hello"; - event.put("foo", foo); - fluency.emit("tag", event); - - assertThat(serialized.get()).isEqualTo(true); - } - - static class Foo - { - String s; + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(sslEnabled); + builder.setAckResponseMode(true); + if (sslEnabled) { + builder.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + } + + try (Fluency fluency = builder.build(serverPort)) { + fluency.emit("foo.bar", new HashMap<>()); + } + }, + 5000); + assertNull(exception); + } + + @ParameterizedTest + @MethodSource("sslFlagsProvider") + void testBufferWithJacksonModule() throws IOException { + AtomicBoolean serialized = new AtomicBoolean(); + SimpleModule simpleModule = new SimpleModule(); + simpleModule.addSerializer(Foo.class, new FooSerializer(serialized)); + + FluentdRecordFormatter.Config recordFormatterConfig = new FluentdRecordFormatter.Config(); + recordFormatterConfig.setJacksonModules(Collections.singletonList(simpleModule)); + + Fluency fluency = + new FluencyBuilder() + .buildFromIngester(new FluentdRecordFormatter(recordFormatterConfig), ingester); + + Map event = new HashMap<>(); + Foo foo = new Foo(); + foo.s = "Hello"; + event.put("foo", foo); + fluency.emit("tag", event); + + assertThat(serialized.get()).isEqualTo(true); + } + + static class Foo { + String s; + } + + public static class FooSerializer extends StdSerializer { + final AtomicBoolean serialized; + + FooSerializer(AtomicBoolean serialized) { + super(Foo.class); + this.serialized = serialized; } - public static class FooSerializer - extends StdSerializer - { - final AtomicBoolean serialized; - - FooSerializer(AtomicBoolean serialized) - { - super(Foo.class); - this.serialized = serialized; - } - - @Override - public void serialize(Foo value, JsonGenerator gen, SerializerProvider provider) - throws IOException - { - gen.writeStartObject(); - gen.writeStringField("s", "Foo:" + value.s); - gen.writeEndObject(); - serialized.set(true); - } + @Override + public void serialize(Foo value, JsonGenerator gen, SerializerProvider provider) + throws IOException { + gen.writeStartObject(); + gen.writeStringField("s", "Foo:" + value.s); + gen.writeEndObject(); + serialized.set(true); } + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyTestWithMockServer.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyTestWithMockServer.java index 59a1bae2..aa91970f 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyTestWithMockServer.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/FluencyTestWithMockServer.java @@ -16,8 +16,31 @@ package org.komamitsu.fluency.fluentd; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; + import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.komamitsu.fluency.EventTime; @@ -37,699 +60,752 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.net.Socket; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Stream; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; - -class FluencyTestWithMockServer -{ - private static final Logger LOG = LoggerFactory.getLogger(FluencyTestWithMockServer.class); - private static final int SMALL_BUF_SIZE = 4 * 1024 * 1024; - private static final String TMPDIR = System.getProperty("java.io.tmpdir"); - - private FailureDetector createFailureDetector(Heartbeater hb) - { - return new FailureDetector(new PhiAccrualFailureDetectStrategy(), hb); - } - - private FluentdSender getSingleTCPSender(int port) - { - TCPSender.Config config = new TCPSender.Config(); - config.setPort(port); - return new TCPSender(config); - } - - private FluentdSender getDoubleTCPSender(int firstPort, int secondPort) - { - TCPSender.Config config0 = new TCPSender.Config(); - config0.setPort(firstPort); - - TCPHeartbeater.Config hbConfig0 = new TCPHeartbeater.Config(); - hbConfig0.setPort(firstPort); - - TCPSender.Config config1 = new TCPSender.Config(); - config1.setPort(secondPort); - - TCPHeartbeater.Config hbConfig1 = new TCPHeartbeater.Config(); - hbConfig1.setPort(secondPort); - - return new MultiSender(new MultiSender.Config(), - ImmutableList.of( - new TCPSender(config0, - createFailureDetector(new TCPHeartbeater(hbConfig0))), - new TCPSender(config1, - createFailureDetector(new TCPHeartbeater(hbConfig1))))); - } - - private FluentdSender getSingleSSLSender(int port) - { - SSLSender.Config config = new SSLSender.Config(); - config.setPort(port); - config.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - return new SSLSender(config); - } - - private FluentdSender getDoubleSSLSender(int firstPort, int secondPort) - { - SSLSender.Config config0 = new SSLSender.Config(); - config0.setPort(firstPort); - config0.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - - TCPHeartbeater.Config hbConfig0 = new TCPHeartbeater.Config(); - hbConfig0.setPort(firstPort); - - SSLSender.Config config1 = new SSLSender.Config(); - config1.setPort(secondPort); - config1.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - - TCPHeartbeater.Config hbConfig1 = new TCPHeartbeater.Config(); - hbConfig1.setPort(secondPort); - - return new MultiSender(new MultiSender.Config(), - ImmutableList.of( - new SSLSender(config0, - createFailureDetector(new TCPHeartbeater(hbConfig0))), - new SSLSender(config1, - createFailureDetector(new TCPHeartbeater(hbConfig1))))); - } - - static Stream optionsProvider() - { - return Stream.of( - // TCP - new Options(false, false, false, false, false, false, false), // Normal - new Options(true, false, false, false, false, false, false), // Failover - new Options(false, true, false, true, false, false, false), // File backup + Ack response - new Options(false, false, true, false, false, false, false), // Close instead of flush - new Options(false, false, false, true, false, false, false), // Ack response - new Options(false, false, false, false, true, false, false), // Small buffer - new Options(false, false, false, false, false, false, true), // JVM heap - new Options(true, false, false, false, true, false, false), // Failover + Small buffer - new Options(false, false, true, false, true, false, false), // Close instead of flush + Small buffer - new Options(false, false, false, true, true, false, false), // Ack response + Small buffer - new Options(true, false, true, false, false, false, false), // Failover + Close instead of flush - new Options(false, true, true, true, false, false, false), // File backup + Ack response + Close instead of flush - new Options(false, false, true, true, false, false, false), // Ack response + Close instead of flush - new Options(false, false, false, false, false, false, false, EmitType.MAP_WITH_EVENT_TIME), // EmitType = MAP_WITH_EVENT_TIME - new Options(false, false, false, false, false, false, false, EmitType.MSGPACK_MAP_VALUE_BYTES), // EmitType = MSGPACK_MAP_VALUE_BYTES - new Options(false, false, false, false, false, false, false, EmitType.MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME), // EmitType = MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME - new Options(false, false, false, false, false, false, false, EmitType.MSGPACK_MAP_VALUE_BYTEBUFFER), // EmitType = MSGPACK_MAP_VALUE_BYTEBUFFER - new Options(false, false, false, false, false, false, false, EmitType.MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME), // EmitType = MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME - // SSL - new Options(false, false, false, false, false, true, false), // Normal - new Options(true, false, false, false, false, true, false), // Failover - new Options(false, true, false, true, false, true, false), // File backup + Ack response - new Options(false, false, true, false, false, true, false), // Close instead of flush - new Options(false, false, false, true, false, true, false), // Ack response - new Options(false, false, false, false, true, true, false), // Small buffer - new Options(false, false, false, false, false, true, true), // JVM heap - new Options(true, false, false, false, true, true, false), // Failover + Small buffer - new Options(false, false, true, false, true, true, false), // Close instead of flush + Small buffer - new Options(false, false, false, true, true, true, false), // Ack response + Small buffer - new Options(true, false, true, false, false, true, false), // Failover + Close instead of flush - new Options(false, true, true, true, false, true, false), // File backup + Ack response + Close instead of flush - new Options(false, false, true, true, false, true, false), // Ack response + Close instead of flush - new Options(false, false, false, false, false, true, false, EmitType.MAP_WITH_EVENT_TIME), // EmitType = MAP_WITH_EVENT_TIME - new Options(false, false, false, false, false, true, false, EmitType.MSGPACK_MAP_VALUE_BYTES), // EmitType = MSGPACK_MAP_VALUE_BYTES - new Options(false, false, false, false, false, true, false, EmitType.MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME), // EmitType = MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME - new Options(false, false, false, false, false, true, false, EmitType.MSGPACK_MAP_VALUE_BYTEBUFFER), // EmitType = MSGPACK_MAP_VALUE_BYTEBUFFER - new Options(false, false, false, false, false, true, false, EmitType.MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME) // EmitType = MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME +class FluencyTestWithMockServer { + private static final Logger LOG = LoggerFactory.getLogger(FluencyTestWithMockServer.class); + private static final int SMALL_BUF_SIZE = 4 * 1024 * 1024; + private static final String TMPDIR = System.getProperty("java.io.tmpdir"); + + private FailureDetector createFailureDetector(Heartbeater hb) { + return new FailureDetector(new PhiAccrualFailureDetectStrategy(), hb); + } + + private FluentdSender getSingleTCPSender(int port) { + TCPSender.Config config = new TCPSender.Config(); + config.setPort(port); + return new TCPSender(config); + } + + private FluentdSender getDoubleTCPSender(int firstPort, int secondPort) { + TCPSender.Config config0 = new TCPSender.Config(); + config0.setPort(firstPort); + + TCPHeartbeater.Config hbConfig0 = new TCPHeartbeater.Config(); + hbConfig0.setPort(firstPort); + + TCPSender.Config config1 = new TCPSender.Config(); + config1.setPort(secondPort); + + TCPHeartbeater.Config hbConfig1 = new TCPHeartbeater.Config(); + hbConfig1.setPort(secondPort); + + return new MultiSender( + new MultiSender.Config(), + ImmutableList.of( + new TCPSender(config0, createFailureDetector(new TCPHeartbeater(hbConfig0))), + new TCPSender(config1, createFailureDetector(new TCPHeartbeater(hbConfig1))))); + } + + private FluentdSender getSingleSSLSender(int port) { + SSLSender.Config config = new SSLSender.Config(); + config.setPort(port); + config.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + return new SSLSender(config); + } + + private FluentdSender getDoubleSSLSender(int firstPort, int secondPort) { + SSLSender.Config config0 = new SSLSender.Config(); + config0.setPort(firstPort); + config0.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + + TCPHeartbeater.Config hbConfig0 = new TCPHeartbeater.Config(); + hbConfig0.setPort(firstPort); + + SSLSender.Config config1 = new SSLSender.Config(); + config1.setPort(secondPort); + config1.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + + TCPHeartbeater.Config hbConfig1 = new TCPHeartbeater.Config(); + hbConfig1.setPort(secondPort); + + return new MultiSender( + new MultiSender.Config(), + ImmutableList.of( + new SSLSender(config0, createFailureDetector(new TCPHeartbeater(hbConfig0))), + new SSLSender(config1, createFailureDetector(new TCPHeartbeater(hbConfig1))))); + } + + static Stream optionsProvider() { + return Stream.of( + // TCP + new Options(false, false, false, false, false, false, false), // Normal + new Options(true, false, false, false, false, false, false), // Failover + new Options(false, true, false, true, false, false, false), // File backup + Ack response + new Options(false, false, true, false, false, false, false), // Close instead of flush + new Options(false, false, false, true, false, false, false), // Ack response + new Options(false, false, false, false, true, false, false), // Small buffer + new Options(false, false, false, false, false, false, true), // JVM heap + new Options(true, false, false, false, true, false, false), // Failover + Small buffer + new Options( + false, false, true, false, true, false, false), // Close instead of flush + Small buffer + new Options(false, false, false, true, true, false, false), // Ack response + Small buffer + new Options( + true, false, true, false, false, false, false), // Failover + Close instead of flush + new Options( + false, true, true, true, false, false, + false), // File backup + Ack response + Close instead of flush + new Options( + false, false, true, true, false, false, false), // Ack response + Close instead of flush + new Options( + false, + false, + false, + false, + false, + false, + false, + EmitType.MAP_WITH_EVENT_TIME), // EmitType = MAP_WITH_EVENT_TIME + new Options( + false, + false, + false, + false, + false, + false, + false, + EmitType.MSGPACK_MAP_VALUE_BYTES), // EmitType = MSGPACK_MAP_VALUE_BYTES + new Options( + false, + false, + false, + false, + false, + false, + false, + EmitType.MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME), // EmitType = + // MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME + new Options( + false, + false, + false, + false, + false, + false, + false, + EmitType.MSGPACK_MAP_VALUE_BYTEBUFFER), // EmitType = MSGPACK_MAP_VALUE_BYTEBUFFER + new Options( + false, + false, + false, + false, + false, + false, + false, + EmitType.MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME), // EmitType = + // MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME + // SSL + new Options(false, false, false, false, false, true, false), // Normal + new Options(true, false, false, false, false, true, false), // Failover + new Options(false, true, false, true, false, true, false), // File backup + Ack response + new Options(false, false, true, false, false, true, false), // Close instead of flush + new Options(false, false, false, true, false, true, false), // Ack response + new Options(false, false, false, false, true, true, false), // Small buffer + new Options(false, false, false, false, false, true, true), // JVM heap + new Options(true, false, false, false, true, true, false), // Failover + Small buffer + new Options( + false, false, true, false, true, true, false), // Close instead of flush + Small buffer + new Options(false, false, false, true, true, true, false), // Ack response + Small buffer + new Options( + true, false, true, false, false, true, false), // Failover + Close instead of flush + new Options( + false, true, true, true, false, true, + false), // File backup + Ack response + Close instead of flush + new Options( + false, false, true, true, false, true, false), // Ack response + Close instead of flush + new Options( + false, + false, + false, + false, + false, + true, + false, + EmitType.MAP_WITH_EVENT_TIME), // EmitType = MAP_WITH_EVENT_TIME + new Options( + false, + false, + false, + false, + false, + true, + false, + EmitType.MSGPACK_MAP_VALUE_BYTES), // EmitType = MSGPACK_MAP_VALUE_BYTES + new Options( + false, + false, + false, + false, + false, + true, + false, + EmitType.MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME), // EmitType = + // MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME + new Options( + false, + false, + false, + false, + false, + true, + false, + EmitType.MSGPACK_MAP_VALUE_BYTEBUFFER), // EmitType = MSGPACK_MAP_VALUE_BYTEBUFFER + new Options( + false, + false, + false, + false, + false, + true, + false, + EmitType.MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME) // EmitType = + // MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME ); - } - - @ParameterizedTest - @MethodSource("optionsProvider") - void testFluencyUsingAsyncFlusher(final Options options) - throws Exception - { - testFluencyBase(localPorts -> { - FluentdSender sender; - int fluentdPort = localPorts.get(0); - if (options.failover) { - int secondaryFluentdPort = localPorts.get(1); - if (options.sslEnabled) { - sender = getDoubleSSLSender(fluentdPort, secondaryFluentdPort); - } - else { - sender = getDoubleTCPSender(fluentdPort, secondaryFluentdPort); - } - } - else { - if (options.sslEnabled) { - sender = getSingleSSLSender(fluentdPort); - } - else { - sender = getSingleTCPSender(fluentdPort); - } - } - - FluentdIngester.Config ingesterConfig = new FluentdIngester.Config(); - if (options.ackResponse) { - ingesterConfig.setAckResponseMode(true); + } + + @ParameterizedTest + @MethodSource("optionsProvider") + void testFluencyUsingAsyncFlusher(final Options options) throws Exception { + testFluencyBase( + localPorts -> { + FluentdSender sender; + int fluentdPort = localPorts.get(0); + if (options.failover) { + int secondaryFluentdPort = localPorts.get(1); + if (options.sslEnabled) { + sender = getDoubleSSLSender(fluentdPort, secondaryFluentdPort); + } else { + sender = getDoubleTCPSender(fluentdPort, secondaryFluentdPort); } - - Buffer.Config bufferConfig = new Buffer.Config(); - if (options.smallBuffer) { - bufferConfig.setChunkRetentionSize(SMALL_BUF_SIZE); - bufferConfig.setMaxBufferSize(SMALL_BUF_SIZE + 1); - } - if (options.fileBackup) { - bufferConfig.setFileBackupDir(TMPDIR); - bufferConfig.setFileBackupPrefix("testFluencyUsingAsyncFlusher" + options.hashCode()); - } - if (options.jvmHeap) { - bufferConfig.setJvmHeapBufferMode(true); + } else { + if (options.sslEnabled) { + sender = getSingleSSLSender(fluentdPort); + } else { + sender = getSingleTCPSender(fluentdPort); } - - Flusher.Config flusherConfig = new Flusher.Config(); - flusherConfig.setWaitUntilBufferFlushed(10); - flusherConfig.setWaitUntilTerminated(10); - - Buffer buffer = new Buffer(bufferConfig, new FluentdRecordFormatter(new FluentdRecordFormatter.Config())); - Flusher flusher = new Flusher(flusherConfig, buffer, new FluentdIngester(ingesterConfig, sender)); - - return new Fluency(buffer, flusher); - }, options); + } + + FluentdIngester.Config ingesterConfig = new FluentdIngester.Config(); + if (options.ackResponse) { + ingesterConfig.setAckResponseMode(true); + } + + Buffer.Config bufferConfig = new Buffer.Config(); + if (options.smallBuffer) { + bufferConfig.setChunkRetentionSize(SMALL_BUF_SIZE); + bufferConfig.setMaxBufferSize(SMALL_BUF_SIZE + 1); + } + if (options.fileBackup) { + bufferConfig.setFileBackupDir(TMPDIR); + bufferConfig.setFileBackupPrefix("testFluencyUsingAsyncFlusher" + options.hashCode()); + } + if (options.jvmHeap) { + bufferConfig.setJvmHeapBufferMode(true); + } + + Flusher.Config flusherConfig = new Flusher.Config(); + flusherConfig.setWaitUntilBufferFlushed(10); + flusherConfig.setWaitUntilTerminated(10); + + Buffer buffer = + new Buffer( + bufferConfig, new FluentdRecordFormatter(new FluentdRecordFormatter.Config())); + Flusher flusher = + new Flusher(flusherConfig, buffer, new FluentdIngester(ingesterConfig, sender)); + + return new Fluency(buffer, flusher); + }, + options); + } + + private void testFluencyBase(final FluencyFactory fluencyFactory, final Options options) + throws Exception { + LOG.info("testFluencyBase starts: options={}", options); + + final ArrayList localPorts = new ArrayList<>(); + + final MockFluentdServer fluentd = new MockFluentdServer(options.sslEnabled); + fluentd.start(); + + final MockFluentdServer secondaryFluentd = new MockFluentdServer(options.sslEnabled, fluentd); + secondaryFluentd.start(); + + localPorts.add(fluentd.getLocalPort()); + localPorts.add(secondaryFluentd.getLocalPort()); + + final AtomicReference fluency = + new AtomicReference<>(fluencyFactory.generate(localPorts)); + if (options.fileBackup) { + fluency.get().clearBackupFiles(); } - private void testFluencyBase(final FluencyFactory fluencyFactory, final Options options) - throws Exception - { - LOG.info("testFluencyBase starts: options={}", options); - - final ArrayList localPorts = new ArrayList<>(); - - final MockFluentdServer fluentd = new MockFluentdServer(options.sslEnabled); - fluentd.start(); - - final MockFluentdServer secondaryFluentd = new MockFluentdServer(options.sslEnabled, fluentd); - secondaryFluentd.start(); - - localPorts.add(fluentd.getLocalPort()); - localPorts.add(secondaryFluentd.getLocalPort()); + final ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); - final AtomicReference fluency = new AtomicReference<>(fluencyFactory.generate(localPorts)); - if (options.fileBackup) { - fluency.get().clearBackupFiles(); - } - - final ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + final int maxNameLen = 200; + final HashMap nameLenTable = new HashMap<>(maxNameLen); + for (int i = 1; i <= maxNameLen; i++) { + StringBuilder stringBuilder = new StringBuilder(); + for (int j = 0; j < i; j++) { + stringBuilder.append('x'); + } + nameLenTable.put(i, stringBuilder.toString()); + } - final int maxNameLen = 200; - final HashMap nameLenTable = new HashMap<>(maxNameLen); - for (int i = 1; i <= maxNameLen; i++) { - StringBuilder stringBuilder = new StringBuilder(); - for (int j = 0; j < i; j++) { - stringBuilder.append('x'); - } - nameLenTable.put(i, stringBuilder.toString()); - } + final AtomicLong ageEventsSum = new AtomicLong(); + final AtomicLong nameEventsLength = new AtomicLong(); + final AtomicLong tag0EventsCounter = new AtomicLong(); + final AtomicLong tag1EventsCounter = new AtomicLong(); + final AtomicLong tag2EventsCounter = new AtomicLong(); + final AtomicLong tag3EventsCounter = new AtomicLong(); + + try { + final Random random = new Random(); + int concurrency = 10; + final int reqNum = 6000; + long start = System.currentTimeMillis(); + final CountDownLatch latch = new CountDownLatch(concurrency); + final AtomicBoolean shouldFailOver = new AtomicBoolean(true); + + final AtomicBoolean shouldStopFluentd = new AtomicBoolean(true); + final AtomicBoolean shouldStopFluency = new AtomicBoolean(true); + final CountDownLatch fluentdCloseWaitLatch = new CountDownLatch(concurrency); + final CountDownLatch fluencyCloseWaitLatch = new CountDownLatch(concurrency); + + ExecutorService es = Executors.newCachedThreadPool(); + for (int i = 0; i < concurrency; i++) { + es.execute( + () -> { + for (int i1 = 0; i1 < reqNum; i1++) { + if (Thread.currentThread().isInterrupted()) { + LOG.info("Interrupted..."); + break; + } - final AtomicLong ageEventsSum = new AtomicLong(); - final AtomicLong nameEventsLength = new AtomicLong(); - final AtomicLong tag0EventsCounter = new AtomicLong(); - final AtomicLong tag1EventsCounter = new AtomicLong(); - final AtomicLong tag2EventsCounter = new AtomicLong(); - final AtomicLong tag3EventsCounter = new AtomicLong(); - - try { - final Random random = new Random(); - int concurrency = 10; - final int reqNum = 6000; - long start = System.currentTimeMillis(); - final CountDownLatch latch = new CountDownLatch(concurrency); - final AtomicBoolean shouldFailOver = new AtomicBoolean(true); - - final AtomicBoolean shouldStopFluentd = new AtomicBoolean(true); - final AtomicBoolean shouldStopFluency = new AtomicBoolean(true); - final CountDownLatch fluentdCloseWaitLatch = new CountDownLatch(concurrency); - final CountDownLatch fluencyCloseWaitLatch = new CountDownLatch(concurrency); - - ExecutorService es = Executors.newCachedThreadPool(); - for (int i = 0; i < concurrency; i++) { - es.execute(() -> { - for (int i1 = 0; i1 < reqNum; i1++) { - if (Thread.currentThread().isInterrupted()) { - LOG.info("Interrupted..."); - break; - } - - if (options.failover) { - if (i1 == reqNum / 2) { - if (shouldFailOver.getAndSet(false)) { - LOG.info("Failing over..."); - try { - secondaryFluentd.stop(); - } - catch (IOException e) { - LOG.warn("Failed to stop secondary fluentd", e); - } - } - } - } - else if (options.fileBackup) { - if (i1 == reqNum / 2) { - if (shouldStopFluentd.getAndSet(false)) { - LOG.info("Stopping Fluentd..."); - try { - fluentd.stop(); - secondaryFluentd.stop(); - } - catch (IOException e) { - LOG.warn("Failed to stop Fluentd", e); - } - } - - fluentdCloseWaitLatch.countDown(); - try { - assertTrue(fluentdCloseWaitLatch.await(20, TimeUnit.SECONDS)); - } - catch (InterruptedException e) { - LOG.warn("Interrupted", e); - } - - if (shouldStopFluency.getAndSet(false)) { - LOG.info("Stopping Fluency..."); - try { - fluency.get().close(); - TimeUnit.SECONDS.sleep(2); - } - catch (Exception e) { - LOG.warn("Failed to stop Fluency", e); - } - - LOG.info("Restarting Fluentd..."); - try { - fluentd.start(); - secondaryFluentd.start(); - LOG.info("Restarting Fluency..."); - fluency.set(fluencyFactory.generate(Arrays.asList(fluentd.getLocalPort(), secondaryFluentd.getLocalPort()))); - TimeUnit.SECONDS.sleep(2); - } - catch (Exception e) { - LOG.warn("Failed to restart Fluentd", e); - } - } - - fluencyCloseWaitLatch.countDown(); - try { - assertTrue(fluencyCloseWaitLatch.await(20, TimeUnit.SECONDS)); - } - catch (InterruptedException e) { - LOG.warn("Interrupted", e); - } - } - } - int tagNum = i1 % 4; - final String tag = String.format("foodb%d.bartbl%d", tagNum, tagNum); - switch (tagNum) { - case 0: - tag0EventsCounter.incrementAndGet(); - break; - case 1: - tag1EventsCounter.incrementAndGet(); - break; - case 2: - tag2EventsCounter.incrementAndGet(); - break; - case 3: - tag3EventsCounter.incrementAndGet(); - break; - default: - throw new RuntimeException("Never reach here"); - } - - int rand = random.nextInt(maxNameLen); - final Map hashMap = new HashMap(); - String name = nameLenTable.get(rand + 1); - nameEventsLength.addAndGet(name.length()); - hashMap.put("name", name); - rand = random.nextInt(100); - int age = rand; - ageEventsSum.addAndGet(age); - hashMap.put("age", age); - hashMap.put("comment", "hello, world"); - hashMap.put("rate", 1.23); - - try { - Exception exception = null; - for (int retry = 0; retry < 10; retry++) { - try { - switch (options.emitType) { - case MAP: { - fluency.get().emit(tag, hashMap); - break; - } - case MAP_WITH_EVENT_TIME: { - EventTime eventTime = EventTime.fromEpochMilli(System.currentTimeMillis()); - fluency.get().emit(tag, eventTime, hashMap); - break; - } - case MSGPACK_MAP_VALUE_BYTES: { - byte[] bytes = objectMapper.writeValueAsBytes(hashMap); - fluency.get().emit(tag, bytes, 0, bytes.length); - break; - } - case MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME: { - EventTime eventTime = EventTime.fromEpochMilli(System.currentTimeMillis()); - byte[] bytes = objectMapper.writeValueAsBytes(hashMap); - fluency.get().emit(tag, eventTime, bytes, 0, bytes.length); - break; - } - case MSGPACK_MAP_VALUE_BYTEBUFFER: { - byte[] bytes = objectMapper.writeValueAsBytes(hashMap); - fluency.get().emit(tag, ByteBuffer.wrap(bytes)); - break; - } - case MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME: { - EventTime eventTime = EventTime.fromEpochMilli(System.currentTimeMillis()); - byte[] bytes = objectMapper.writeValueAsBytes(hashMap); - fluency.get().emit(tag, eventTime, ByteBuffer.wrap(bytes)); - break; - } - } - exception = null; - break; - } - catch (Exception e) { - exception = e; - try { - TimeUnit.SECONDS.sleep(1); - } - catch (InterruptedException e1) { - } - } - } - if (exception != null) { - throw exception; - } - } - catch (Exception e) { - LOG.warn("Exception occurred", e); - } + if (options.failover) { + if (i1 == reqNum / 2) { + if (shouldFailOver.getAndSet(false)) { + LOG.info("Failing over..."); + try { + secondaryFluentd.stop(); + } catch (IOException e) { + LOG.warn("Failed to stop secondary fluentd", e); + } + } + } + } else if (options.fileBackup) { + if (i1 == reqNum / 2) { + if (shouldStopFluentd.getAndSet(false)) { + LOG.info("Stopping Fluentd..."); + try { + fluentd.stop(); + secondaryFluentd.stop(); + } catch (IOException e) { + LOG.warn("Failed to stop Fluentd", e); + } } - latch.countDown(); - }); - } - - if (!latch.await(60, TimeUnit.SECONDS)) { - fail("Sending all requests is timed out"); - } - - if (options.closeInsteadOfFlush) { - fluency.get().close(); - } - else { - fluency.get().flush(); - fluency.get().waitUntilAllBufferFlushed(20); - } - fluentd.waitUntilEventsStop(); - fluentd.stop(); + fluentdCloseWaitLatch.countDown(); + try { + assertTrue(fluentdCloseWaitLatch.await(20, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + LOG.warn("Interrupted", e); + } - secondaryFluentd.waitUntilEventsStop(); - secondaryFluentd.stop(); + if (shouldStopFluency.getAndSet(false)) { + LOG.info("Stopping Fluency..."); + try { + fluency.get().close(); + TimeUnit.SECONDS.sleep(2); + } catch (Exception e) { + LOG.warn("Failed to stop Fluency", e); + } + + LOG.info("Restarting Fluentd..."); + try { + fluentd.start(); + secondaryFluentd.start(); + LOG.info("Restarting Fluency..."); + fluency.set( + fluencyFactory.generate( + Arrays.asList( + fluentd.getLocalPort(), secondaryFluentd.getLocalPort()))); + TimeUnit.SECONDS.sleep(2); + } catch (Exception e) { + LOG.warn("Failed to restart Fluentd", e); + } + } - if (options.failover) { - assertThat(fluentd.connectCounter.get()).isGreaterThan(0L); - assertThat(fluentd.connectCounter.get()).isLessThanOrEqualTo(10L); - assertThat(fluentd.closeCounter.get()).isGreaterThan(0L); - assertThat(fluentd.closeCounter.get()).isLessThanOrEqualTo(10L); - } - else { - assertThat(fluentd.connectCounter.get()).isGreaterThan(0L); - assertThat(fluentd.connectCounter.get()).isLessThanOrEqualTo(2L); - if (options.closeInsteadOfFlush) { - assertThat(fluentd.closeCounter.get()).isGreaterThan(0L); + fluencyCloseWaitLatch.countDown(); + try { + assertTrue(fluencyCloseWaitLatch.await(20, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + LOG.warn("Interrupted", e); + } + } } - else { - assertThat(fluentd.closeCounter.get()).isEqualTo(0L); + int tagNum = i1 % 4; + final String tag = String.format("foodb%d.bartbl%d", tagNum, tagNum); + switch (tagNum) { + case 0: + tag0EventsCounter.incrementAndGet(); + break; + case 1: + tag1EventsCounter.incrementAndGet(); + break; + case 2: + tag2EventsCounter.incrementAndGet(); + break; + case 3: + tag3EventsCounter.incrementAndGet(); + break; + default: + throw new RuntimeException("Never reach here"); } - assertThat(fluentd.closeCounter.get()).isLessThanOrEqualTo(2L); - } - assertEquals((long) concurrency * reqNum, fluentd.ageEventsCounter.get()); - assertEquals(ageEventsSum.get(), fluentd.ageEventsSum.get()); - assertEquals((long) concurrency * reqNum, fluentd.nameEventsCounter.get()); - assertEquals(nameEventsLength.get(), fluentd.nameEventsLength.get()); - assertEquals(tag0EventsCounter.get(), fluentd.tag0EventsCounter.get()); - assertEquals(tag1EventsCounter.get(), fluentd.tag1EventsCounter.get()); - assertEquals(tag2EventsCounter.get(), fluentd.tag2EventsCounter.get()); - assertEquals(tag3EventsCounter.get(), fluentd.tag3EventsCounter.get()); - - System.out.println(System.currentTimeMillis() - start); - } - finally { - fluency.get().close(); - fluentd.stop(); - secondaryFluentd.stop(); + int rand = random.nextInt(maxNameLen); + final Map hashMap = new HashMap(); + String name = nameLenTable.get(rand + 1); + nameEventsLength.addAndGet(name.length()); + hashMap.put("name", name); + rand = random.nextInt(100); + int age = rand; + ageEventsSum.addAndGet(age); + hashMap.put("age", age); + hashMap.put("comment", "hello, world"); + hashMap.put("rate", 1.23); + + try { + Exception exception = null; + for (int retry = 0; retry < 10; retry++) { + try { + switch (options.emitType) { + case MAP: + { + fluency.get().emit(tag, hashMap); + break; + } + case MAP_WITH_EVENT_TIME: + { + EventTime eventTime = + EventTime.fromEpochMilli(System.currentTimeMillis()); + fluency.get().emit(tag, eventTime, hashMap); + break; + } + case MSGPACK_MAP_VALUE_BYTES: + { + byte[] bytes = objectMapper.writeValueAsBytes(hashMap); + fluency.get().emit(tag, bytes, 0, bytes.length); + break; + } + case MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME: + { + EventTime eventTime = + EventTime.fromEpochMilli(System.currentTimeMillis()); + byte[] bytes = objectMapper.writeValueAsBytes(hashMap); + fluency.get().emit(tag, eventTime, bytes, 0, bytes.length); + break; + } + case MSGPACK_MAP_VALUE_BYTEBUFFER: + { + byte[] bytes = objectMapper.writeValueAsBytes(hashMap); + fluency.get().emit(tag, ByteBuffer.wrap(bytes)); + break; + } + case MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME: + { + EventTime eventTime = + EventTime.fromEpochMilli(System.currentTimeMillis()); + byte[] bytes = objectMapper.writeValueAsBytes(hashMap); + fluency.get().emit(tag, eventTime, ByteBuffer.wrap(bytes)); + break; + } + } + exception = null; + break; + } catch (Exception e) { + exception = e; + try { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e1) { + } + } + } + if (exception != null) { + throw exception; + } + } catch (Exception e) { + LOG.warn("Exception occurred", e); + } + } + latch.countDown(); + }); + } + + if (!latch.await(60, TimeUnit.SECONDS)) { + fail("Sending all requests is timed out"); + } + + if (options.closeInsteadOfFlush) { + fluency.get().close(); + } else { + fluency.get().flush(); + fluency.get().waitUntilAllBufferFlushed(20); + } + + fluentd.waitUntilEventsStop(); + fluentd.stop(); + + secondaryFluentd.waitUntilEventsStop(); + secondaryFluentd.stop(); + + if (options.failover) { + assertThat(fluentd.connectCounter.get()).isGreaterThan(0L); + assertThat(fluentd.connectCounter.get()).isLessThanOrEqualTo(10L); + assertThat(fluentd.closeCounter.get()).isGreaterThan(0L); + assertThat(fluentd.closeCounter.get()).isLessThanOrEqualTo(10L); + } else { + assertThat(fluentd.connectCounter.get()).isGreaterThan(0L); + assertThat(fluentd.connectCounter.get()).isLessThanOrEqualTo(2L); + if (options.closeInsteadOfFlush) { + assertThat(fluentd.closeCounter.get()).isGreaterThan(0L); + } else { + assertThat(fluentd.closeCounter.get()).isEqualTo(0L); } + assertThat(fluentd.closeCounter.get()).isLessThanOrEqualTo(2L); + } + + assertEquals((long) concurrency * reqNum, fluentd.ageEventsCounter.get()); + assertEquals(ageEventsSum.get(), fluentd.ageEventsSum.get()); + assertEquals((long) concurrency * reqNum, fluentd.nameEventsCounter.get()); + assertEquals(nameEventsLength.get(), fluentd.nameEventsLength.get()); + assertEquals(tag0EventsCounter.get(), fluentd.tag0EventsCounter.get()); + assertEquals(tag1EventsCounter.get(), fluentd.tag1EventsCounter.get()); + assertEquals(tag2EventsCounter.get(), fluentd.tag2EventsCounter.get()); + assertEquals(tag3EventsCounter.get(), fluentd.tag3EventsCounter.get()); + + System.out.println(System.currentTimeMillis() - start); + } finally { + fluency.get().close(); + fluentd.stop(); + secondaryFluentd.stop(); } - - private enum EmitType - { - MAP, MAP_WITH_EVENT_TIME, - MSGPACK_MAP_VALUE_BYTES, MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME, - MSGPACK_MAP_VALUE_BYTEBUFFER, MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME, + } + + private enum EmitType { + MAP, + MAP_WITH_EVENT_TIME, + MSGPACK_MAP_VALUE_BYTES, + MSGPACK_MAP_VALUE_BYTES_WITH_EVENT_TIME, + MSGPACK_MAP_VALUE_BYTEBUFFER, + MSGPACK_MAP_VALUE_BYTEBUFFER_WITH_EVENT_TIME, + } + + interface FluencyFactory { + Fluency generate(List localPort) throws IOException; + } + + public static class Options { + private final boolean failover; + private final boolean fileBackup; + private final boolean closeInsteadOfFlush; + private final boolean ackResponse; + private final boolean smallBuffer; + private final boolean sslEnabled; + private final boolean jvmHeap; + private final EmitType emitType; + + Options( + boolean failover, + boolean fileBackup, + boolean closeInsteadOfFlush, + boolean ackResponse, + boolean smallBuffer, + boolean sslEnabled, + boolean jvmHeap) { + this( + failover, + fileBackup, + closeInsteadOfFlush, + ackResponse, + smallBuffer, + sslEnabled, + jvmHeap, + EmitType.MAP); } - interface FluencyFactory - { - Fluency generate(List localPort) - throws IOException; + Options( + boolean failover, + boolean fileBackup, + boolean closeInsteadOfFlush, + boolean ackResponse, + boolean smallBuffer, + boolean sslEnabled, + boolean jvmHeap, + EmitType emitType) { + this.failover = failover; + this.fileBackup = fileBackup; + this.closeInsteadOfFlush = closeInsteadOfFlush; + this.ackResponse = ackResponse; + this.smallBuffer = smallBuffer; + this.sslEnabled = sslEnabled; + this.jvmHeap = jvmHeap; + this.emitType = emitType; } - public static class Options - { - private final boolean failover; - private final boolean fileBackup; - private final boolean closeInsteadOfFlush; - private final boolean ackResponse; - private final boolean smallBuffer; - private final boolean sslEnabled; - private final boolean jvmHeap; - private final EmitType emitType; - - Options( - boolean failover, - boolean fileBackup, - boolean closeInsteadOfFlush, - boolean ackResponse, - boolean smallBuffer, - boolean sslEnabled, - boolean jvmHeap) - { - this(failover, fileBackup, closeInsteadOfFlush, ackResponse, smallBuffer, sslEnabled, jvmHeap, EmitType.MAP); - } - - Options( - boolean failover, - boolean fileBackup, - boolean closeInsteadOfFlush, - boolean ackResponse, - boolean smallBuffer, - boolean sslEnabled, - boolean jvmHeap, - EmitType emitType) - { - this.failover = failover; - this.fileBackup = fileBackup; - this.closeInsteadOfFlush = closeInsteadOfFlush; - this.ackResponse = ackResponse; - this.smallBuffer = smallBuffer; - this.sslEnabled = sslEnabled; - this.jvmHeap = jvmHeap; - this.emitType = emitType; - } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Options)) { + return false; + } + + Options options = (Options) o; + + if (failover != options.failover) { + return false; + } + if (fileBackup != options.fileBackup) { + return false; + } + if (closeInsteadOfFlush != options.closeInsteadOfFlush) { + return false; + } + if (ackResponse != options.ackResponse) { + return false; + } + if (smallBuffer != options.smallBuffer) { + return false; + } + if (sslEnabled != options.sslEnabled) { + return false; + } + return emitType == options.emitType; + } - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (!(o instanceof Options)) { - return false; - } + @Override + public int hashCode() { + int result = (failover ? 1 : 0); + result = 31 * result + (fileBackup ? 1 : 0); + result = 31 * result + (closeInsteadOfFlush ? 1 : 0); + result = 31 * result + (ackResponse ? 1 : 0); + result = 31 * result + (smallBuffer ? 1 : 0); + result = 31 * result + (sslEnabled ? 1 : 0); + result = 31 * result + (emitType != null ? emitType.hashCode() : 0); + return result; + } - Options options = (Options) o; + @Override + public String toString() { + return "Options{" + + "failover=" + + failover + + ", fileBackup=" + + fileBackup + + ", closeInsteadOfFlush=" + + closeInsteadOfFlush + + ", ackResponse=" + + ackResponse + + ", smallBuffer=" + + smallBuffer + + ", sslEnabled=" + + sslEnabled + + ", emitType=" + + emitType + + '}'; + } + } + + private static class MockFluentdServer extends AbstractFluentdServer { + private final AtomicLong connectCounter; + private final AtomicLong ageEventsCounter; + private final AtomicLong ageEventsSum; + private final AtomicLong nameEventsCounter; + private final AtomicLong nameEventsLength; + private final AtomicLong tag0EventsCounter; + private final AtomicLong tag1EventsCounter; + private final AtomicLong tag2EventsCounter; + private final AtomicLong tag3EventsCounter; + private final AtomicLong closeCounter; + private final long startTimestampMillis; + + MockFluentdServer(boolean useSsl) throws Exception { + super(useSsl); + connectCounter = new AtomicLong(); + ageEventsCounter = new AtomicLong(); + ageEventsSum = new AtomicLong(); + nameEventsCounter = new AtomicLong(); + nameEventsLength = new AtomicLong(); + tag0EventsCounter = new AtomicLong(); + tag1EventsCounter = new AtomicLong(); + tag2EventsCounter = new AtomicLong(); + tag3EventsCounter = new AtomicLong(); + closeCounter = new AtomicLong(); + startTimestampMillis = System.currentTimeMillis(); + } - if (failover != options.failover) { - return false; - } - if (fileBackup != options.fileBackup) { - return false; - } - if (closeInsteadOfFlush != options.closeInsteadOfFlush) { - return false; - } - if (ackResponse != options.ackResponse) { - return false; - } - if (smallBuffer != options.smallBuffer) { - return false; - } - if (sslEnabled != options.sslEnabled) { - return false; - } - return emitType == options.emitType; - } + MockFluentdServer(boolean useSsl, MockFluentdServer base) throws Exception { + super(useSsl); + connectCounter = base.connectCounter; + ageEventsCounter = base.ageEventsCounter; + ageEventsSum = base.ageEventsSum; + nameEventsCounter = base.nameEventsCounter; + nameEventsLength = base.nameEventsLength; + tag0EventsCounter = base.tag0EventsCounter; + tag1EventsCounter = base.tag1EventsCounter; + tag2EventsCounter = base.tag2EventsCounter; + tag3EventsCounter = base.tag3EventsCounter; + closeCounter = base.closeCounter; + startTimestampMillis = System.currentTimeMillis(); + } + @Override + protected EventHandler getFluentdEventHandler() { + return new EventHandler() { @Override - public int hashCode() - { - int result = (failover ? 1 : 0); - result = 31 * result + (fileBackup ? 1 : 0); - result = 31 * result + (closeInsteadOfFlush ? 1 : 0); - result = 31 * result + (ackResponse ? 1 : 0); - result = 31 * result + (smallBuffer ? 1 : 0); - result = 31 * result + (sslEnabled ? 1 : 0); - result = 31 * result + (emitType != null ? emitType.hashCode() : 0); - return result; + public void onConnect(Socket acceptSocket) { + connectCounter.incrementAndGet(); } @Override - public String toString() - { - return "Options{" + - "failover=" + failover + - ", fileBackup=" + fileBackup + - ", closeInsteadOfFlush=" + closeInsteadOfFlush + - ", ackResponse=" + ackResponse + - ", smallBuffer=" + smallBuffer + - ", sslEnabled=" + sslEnabled + - ", emitType=" + emitType + - '}'; - } - } - - private static class MockFluentdServer - extends AbstractFluentdServer - { - private final AtomicLong connectCounter; - private final AtomicLong ageEventsCounter; - private final AtomicLong ageEventsSum; - private final AtomicLong nameEventsCounter; - private final AtomicLong nameEventsLength; - private final AtomicLong tag0EventsCounter; - private final AtomicLong tag1EventsCounter; - private final AtomicLong tag2EventsCounter; - private final AtomicLong tag3EventsCounter; - private final AtomicLong closeCounter; - private final long startTimestampMillis; - - MockFluentdServer(boolean useSsl) - throws Exception - { - super(useSsl); - connectCounter = new AtomicLong(); - ageEventsCounter = new AtomicLong(); - ageEventsSum = new AtomicLong(); - nameEventsCounter = new AtomicLong(); - nameEventsLength = new AtomicLong(); - tag0EventsCounter = new AtomicLong(); - tag1EventsCounter = new AtomicLong(); - tag2EventsCounter = new AtomicLong(); - tag3EventsCounter = new AtomicLong(); - closeCounter = new AtomicLong(); - startTimestampMillis = System.currentTimeMillis(); - } - - MockFluentdServer(boolean useSsl, MockFluentdServer base) - throws Exception - { - super(useSsl); - connectCounter = base.connectCounter; - ageEventsCounter = base.ageEventsCounter; - ageEventsSum = base.ageEventsSum; - nameEventsCounter = base.nameEventsCounter; - nameEventsLength = base.nameEventsLength; - tag0EventsCounter = base.tag0EventsCounter; - tag1EventsCounter = base.tag1EventsCounter; - tag2EventsCounter = base.tag2EventsCounter; - tag3EventsCounter = base.tag3EventsCounter; - closeCounter = base.closeCounter; - startTimestampMillis = System.currentTimeMillis(); + public void onReceive(String tag, long timestampMillis, MapValue data) { + if (tag.equals("foodb0.bartbl0")) { + tag0EventsCounter.incrementAndGet(); + } else if (tag.equals("foodb1.bartbl1")) { + tag1EventsCounter.incrementAndGet(); + } else if (tag.equals("foodb2.bartbl2")) { + tag2EventsCounter.incrementAndGet(); + } else if (tag.equals("foodb3.bartbl3")) { + tag3EventsCounter.incrementAndGet(); + } else { + throw new IllegalArgumentException("Unexpected tag: tag=" + tag); + } + assertTrue( + startTimestampMillis / 1000 <= timestampMillis / 1000 + && timestampMillis < startTimestampMillis + 60 * 1000); + + assertEquals(4, data.size()); + for (Map.Entry kv : data.entrySet()) { + String key = kv.getKey().asStringValue().toString(); + Value val = kv.getValue(); + if (key.equals("comment")) { + assertEquals("hello, world", val.toString()); + } else if (key.equals("rate")) { + assertEquals(1.23, val.asFloatValue().toFloat(), 0.000001); + } else if (key.equals("name")) { + nameEventsCounter.incrementAndGet(); + nameEventsLength.addAndGet(val.asRawValue().asString().length()); + } else if (key.equals("age")) { + ageEventsCounter.incrementAndGet(); + ageEventsSum.addAndGet(val.asIntegerValue().asInt()); + } + } } @Override - protected EventHandler getFluentdEventHandler() - { - return new EventHandler() - { - @Override - public void onConnect(Socket acceptSocket) - { - connectCounter.incrementAndGet(); - } - - @Override - public void onReceive(String tag, long timestampMillis, MapValue data) - { - if (tag.equals("foodb0.bartbl0")) { - tag0EventsCounter.incrementAndGet(); - } - else if (tag.equals("foodb1.bartbl1")) { - tag1EventsCounter.incrementAndGet(); - } - else if (tag.equals("foodb2.bartbl2")) { - tag2EventsCounter.incrementAndGet(); - } - else if (tag.equals("foodb3.bartbl3")) { - tag3EventsCounter.incrementAndGet(); - } - else { - throw new IllegalArgumentException("Unexpected tag: tag=" + tag); - } - assertTrue(startTimestampMillis / 1000 <= timestampMillis / 1000 && timestampMillis < startTimestampMillis + 60 * 1000); - - assertEquals(4, data.size()); - for (Map.Entry kv : data.entrySet()) { - String key = kv.getKey().asStringValue().toString(); - Value val = kv.getValue(); - if (key.equals("comment")) { - assertEquals("hello, world", val.toString()); - } - else if (key.equals("rate")) { - assertEquals(1.23, val.asFloatValue().toFloat(), 0.000001); - } - else if (key.equals("name")) { - nameEventsCounter.incrementAndGet(); - nameEventsLength.addAndGet(val.asRawValue().asString().length()); - } - else if (key.equals("age")) { - ageEventsCounter.incrementAndGet(); - ageEventsSum.addAndGet(val.asIntegerValue().asInt()); - } - } - } - - @Override - public void onClose(Socket accpetSocket) - { - closeCounter.incrementAndGet(); - } - }; + public void onClose(Socket accpetSocket) { + closeCounter.incrementAndGet(); } + }; } + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/MockTCPServer.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/MockTCPServer.java index f258808f..c8868608 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/MockTCPServer.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/MockTCPServer.java @@ -16,11 +16,6 @@ package org.komamitsu.fluency.fluentd; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.net.ssl.SSLHandshakeException; - import java.io.EOFException; import java.io.IOException; import java.net.ServerSocket; @@ -35,303 +30,283 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import javax.net.ssl.SSLHandshakeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class MockTCPServer -{ - private static final Logger LOG = LoggerFactory.getLogger(MockTCPServer.class); - private final boolean sslEnabled; - private final AtomicLong lastEventTimeStampMilli = new AtomicLong(); - private final AtomicInteger threadSeqNum = new AtomicInteger(); - private ExecutorService executorService; - private ServerTask serverTask; - // TODO Make this class immutable - private List tasks = new ArrayList<>(); - - public MockTCPServer(boolean sslEnabled) - { - this.sslEnabled = sslEnabled; +public class MockTCPServer { + private static final Logger LOG = LoggerFactory.getLogger(MockTCPServer.class); + private final boolean sslEnabled; + private final AtomicLong lastEventTimeStampMilli = new AtomicLong(); + private final AtomicInteger threadSeqNum = new AtomicInteger(); + private ExecutorService executorService; + private ServerTask serverTask; + // TODO Make this class immutable + private List tasks = new ArrayList<>(); + + public MockTCPServer(boolean sslEnabled) { + this.sslEnabled = sslEnabled; + } + + protected EventHandler getEventHandler() { + return new EventHandler() { + @Override + public void onConnect(Socket acceptSocket) {} + + @Override + public void onReceive(Socket acceptSocket, int len, byte[] data) {} + + @Override + public void onClose(Socket acceptSocket) {} + }; + } + + public synchronized void start() throws Exception { + if (executorService == null) { + this.executorService = + Executors.newCachedThreadPool( + new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + return new Thread( + r, String.format("accepted-socket-worker-%d", threadSeqNum.getAndAdd(1))); + } + }); } - protected EventHandler getEventHandler() - { - return new EventHandler() - { - @Override - public void onConnect(Socket acceptSocket) - { - } - - @Override - public void onReceive(Socket acceptSocket, int len, byte[] data) - { - } + if (serverTask == null) { + serverTask = + new ServerTask( + executorService, + lastEventTimeStampMilli, + getEventHandler(), + sslEnabled ? SSLTestSocketFactories.createServerSocket() : new ServerSocket(), + tasks); + executorService.execute(serverTask); + tasks.add(serverTask); + } - @Override - public void onClose(Socket acceptSocket) - { - } - }; + for (int i = 0; i < 10; i++) { + int localPort = serverTask.getLocalPort(); + if (localPort > 0) { + return; + } + TimeUnit.MILLISECONDS.sleep(500); + } + throw new IllegalStateException("Local port open timeout"); + } + + public void waitUntilEventsStop() throws InterruptedException { + for (int i = 0; i < 20; i++) { + if (lastEventTimeStampMilli.get() + 2000 < System.currentTimeMillis()) { + return; + } + TimeUnit.MILLISECONDS.sleep(500); } + throw new IllegalStateException("Events didn't stop in the expected time"); + } - public synchronized void start() - throws Exception - { - if (executorService == null) { - this.executorService = Executors.newCachedThreadPool(new ThreadFactory() - { - @Override - public Thread newThread(Runnable r) - { - return new Thread(r, String.format("accepted-socket-worker-%d", threadSeqNum.getAndAdd(1))); - } - }); - } + @Override + public String toString() { + return "MockTCPServer{" + "serverTask=" + serverTask + ", sslEnabled=" + sslEnabled + '}'; + } - if (serverTask == null) { - serverTask = new ServerTask(executorService, lastEventTimeStampMilli, getEventHandler(), - sslEnabled ? SSLTestSocketFactories.createServerSocket() : new ServerSocket(), tasks); - executorService.execute(serverTask); - tasks.add(serverTask); - } + public int getLocalPort() { + return serverTask.getLocalPort(); + } - for (int i = 0; i < 10; i++) { - int localPort = serverTask.getLocalPort(); - if (localPort > 0) { - return; - } - TimeUnit.MILLISECONDS.sleep(500); - } - throw new IllegalStateException("Local port open timeout"); + public synchronized void stop() throws IOException { + stop(false); + } + + public synchronized void stop(boolean immediate) throws IOException { + if (executorService == null) { + return; + } + LOG.debug("Stopping MockTCPServer... {}", this); + executorService.shutdown(); + try { + if (!executorService.awaitTermination(1000, TimeUnit.MILLISECONDS)) { + LOG.debug("Shutting down MockTCPServer and child tasks... {}", this); + executorService.shutdownNow(); + } + } catch (InterruptedException e) { + LOG.warn("ExecutorService.shutdown() was failed: {}", this, e); + Thread.currentThread().interrupt(); } - public void waitUntilEventsStop() - throws InterruptedException - { - for (int i = 0; i < 20; i++) { - if (lastEventTimeStampMilli.get() + 2000 < System.currentTimeMillis()) { - return; - } - TimeUnit.MILLISECONDS.sleep(500); + if (immediate) { + LOG.debug("Closing related sockets {}", this); + for (Runnable runnable : tasks) { + if (runnable instanceof ServerTask) { + ((ServerTask) runnable).close(); + } else if (runnable instanceof ServerTask.AcceptTask) { + ((ServerTask.AcceptTask) runnable).close(); } - throw new IllegalStateException("Events didn't stop in the expected time"); + } } - @Override - public String toString() - { - return "MockTCPServer{" + - "serverTask=" + serverTask + - ", sslEnabled=" + sslEnabled + - '}'; + executorService = null; + serverTask = null; + } + + public interface EventHandler { + void onConnect(Socket acceptSocket); + + void onReceive(Socket acceptSocket, int len, byte[] data); + + void onClose(Socket acceptSocket); + } + + private static class ServerTask implements Runnable { + private final ServerSocket serverSocket; + private final ExecutorService serverExecutorService; + private final EventHandler eventHandler; + private final AtomicLong lastEventTimeStampMilli; + private final List tasks; + + private ServerTask( + ExecutorService executorService, + AtomicLong lastEventTimeStampMilli, + EventHandler eventHandler, + ServerSocket serverSocket, + List tasks) + throws IOException { + this.serverExecutorService = executorService; + this.lastEventTimeStampMilli = lastEventTimeStampMilli; + this.eventHandler = eventHandler; + this.serverSocket = serverSocket; + if (!serverSocket.isBound()) { + serverSocket.bind(null); + } + this.tasks = tasks; } - public int getLocalPort() - { - return serverTask.getLocalPort(); + public int getLocalPort() { + return serverSocket.getLocalPort(); } - public synchronized void stop() - throws IOException - { - stop(false); + @Override + public String toString() { + return "ServerTask{" + "serverSocket=" + serverSocket + '}'; } - public synchronized void stop(boolean immediate) - throws IOException - { - if (executorService == null) { - return; - } - LOG.debug("Stopping MockTCPServer... {}", this); - executorService.shutdown(); + @Override + public void run() { + while (!serverExecutorService.isShutdown()) { try { - if (!executorService.awaitTermination(1000, TimeUnit.MILLISECONDS)) { - LOG.debug("Shutting down MockTCPServer and child tasks... {}", this); - executorService.shutdownNow(); - } - } - catch (InterruptedException e) { - LOG.warn("ExecutorService.shutdown() was failed: {}", this, e); - Thread.currentThread().interrupt(); - } - - if (immediate) { - LOG.debug("Closing related sockets {}", this); - for (Runnable runnable : tasks) { - if (runnable instanceof ServerTask) { - ((ServerTask) runnable).close(); - } else if (runnable instanceof ServerTask.AcceptTask) { - ((ServerTask.AcceptTask) runnable).close(); - } - } + LOG.debug("ServerTask: accepting... this={}, local.port={}", this, getLocalPort()); + Socket acceptSocket = serverSocket.accept(); + LOG.debug( + "ServerTask: accepted. this={}, local.port={}, remote.port={}", + this, + getLocalPort(), + acceptSocket.getPort()); + AcceptTask acceptTask = + new AcceptTask( + serverExecutorService, lastEventTimeStampMilli, eventHandler, acceptSocket); + serverExecutorService.execute(acceptTask); + tasks.add(acceptTask); + } catch (RejectedExecutionException e) { + LOG.debug("ServerTask: ServerSocket.accept() failed[{}]: this={}", e.getMessage(), this); + } catch (ClosedByInterruptException e) { + LOG.debug("ServerTask: ServerSocket.accept() failed[{}]: this={}", e.getMessage(), this); + } catch (IOException e) { + LOG.warn("ServerTask: ServerSocket.accept() failed[{}]: this={}", e.getMessage(), this); } - - executorService = null; - serverTask = null; + } + try { + serverSocket.close(); + } catch (IOException e) { + LOG.warn("ServerTask: ServerSocketChannel.close() failed", e); + } + LOG.info("ServerTask: Finishing ServerTask...: this={}", this); } - public interface EventHandler - { - void onConnect(Socket acceptSocket); - - void onReceive(Socket acceptSocket, int len, byte[] data); - - void onClose(Socket acceptSocket); + private void close() throws IOException { + serverSocket.close(); } - private static class ServerTask - implements Runnable - { - private final ServerSocket serverSocket; - private final ExecutorService serverExecutorService; - private final EventHandler eventHandler; - private final AtomicLong lastEventTimeStampMilli; - private final List tasks; - - private ServerTask( - ExecutorService executorService, - AtomicLong lastEventTimeStampMilli, - EventHandler eventHandler, - ServerSocket serverSocket, - List tasks - ) - throws IOException - { - this.serverExecutorService = executorService; - this.lastEventTimeStampMilli = lastEventTimeStampMilli; - this.eventHandler = eventHandler; - this.serverSocket = serverSocket; - if (!serverSocket.isBound()) { - serverSocket.bind(null); - } - this.tasks = tasks; - } - - public int getLocalPort() - { - return serverSocket.getLocalPort(); - } - - @Override - public String toString() - { - return "ServerTask{" + - "serverSocket=" + serverSocket + - '}'; - } - - @Override - public void run() - { - while (!serverExecutorService.isShutdown()) { - try { - LOG.debug("ServerTask: accepting... this={}, local.port={}", this, getLocalPort()); - Socket acceptSocket = serverSocket.accept(); - LOG.debug("ServerTask: accepted. this={}, local.port={}, remote.port={}", this, getLocalPort(), acceptSocket.getPort()); - AcceptTask acceptTask = new AcceptTask(serverExecutorService, lastEventTimeStampMilli, eventHandler, acceptSocket); - serverExecutorService.execute(acceptTask); - tasks.add(acceptTask); - } - catch (RejectedExecutionException e) { - LOG.debug("ServerTask: ServerSocket.accept() failed[{}]: this={}", e.getMessage(), this); - } - catch (ClosedByInterruptException e) { - LOG.debug("ServerTask: ServerSocket.accept() failed[{}]: this={}", e.getMessage(), this); - } - catch (IOException e) { - LOG.warn("ServerTask: ServerSocket.accept() failed[{}]: this={}", e.getMessage(), this); - } - } + private static class AcceptTask implements Runnable { + private final Socket acceptSocket; + private final EventHandler eventHandler; + private final ExecutorService serverExecutorService; + private final AtomicLong lastEventTimeStampMilli; + + private AcceptTask( + ExecutorService serverExecutorService, + AtomicLong lastEventTimeStampMilli, + EventHandler eventHandler, + Socket acceptSocket) { + this.serverExecutorService = serverExecutorService; + this.lastEventTimeStampMilli = lastEventTimeStampMilli; + this.eventHandler = eventHandler; + this.acceptSocket = acceptSocket; + } + + private void close() throws IOException { + acceptSocket.close(); + } + + @Override + public void run() { + LOG.debug( + "AcceptTask: connected. this={}, local={}, remote={}", + this, + acceptSocket.getLocalPort(), + acceptSocket.getPort()); + try { + eventHandler.onConnect(acceptSocket); + byte[] byteBuf = new byte[512 * 1024]; + while (!serverExecutorService.isShutdown()) { try { - serverSocket.close(); - } - catch (IOException e) { - LOG.warn("ServerTask: ServerSocketChannel.close() failed", e); - } - LOG.info("ServerTask: Finishing ServerTask...: this={}", this); - } - - private void close() - throws IOException - { - serverSocket.close(); - } - - private static class AcceptTask - implements Runnable - { - private final Socket acceptSocket; - private final EventHandler eventHandler; - private final ExecutorService serverExecutorService; - private final AtomicLong lastEventTimeStampMilli; - - private AcceptTask(ExecutorService serverExecutorService, AtomicLong lastEventTimeStampMilli, EventHandler eventHandler, Socket acceptSocket) - { - this.serverExecutorService = serverExecutorService; - this.lastEventTimeStampMilli = lastEventTimeStampMilli; - this.eventHandler = eventHandler; - this.acceptSocket = acceptSocket; - } - - private void close() - throws IOException - { - acceptSocket.close(); - } - - @Override - public void run() - { - LOG.debug("AcceptTask: connected. this={}, local={}, remote={}", - this, acceptSocket.getLocalPort(), acceptSocket.getPort()); + int len = acceptSocket.getInputStream().read(byteBuf); + if (len <= 0) { + LOG.debug( + "AcceptTask: closed. this={}, local={}, remote={}", + this, + acceptSocket.getLocalPort(), + acceptSocket.getPort()); + eventHandler.onClose(acceptSocket); try { - eventHandler.onConnect(acceptSocket); - byte[] byteBuf = new byte[512 * 1024]; - while (!serverExecutorService.isShutdown()) { - try { - int len = acceptSocket.getInputStream().read(byteBuf); - if (len <= 0) { - LOG.debug("AcceptTask: closed. this={}, local={}, remote={}", - this, acceptSocket.getLocalPort(), acceptSocket.getPort()); - eventHandler.onClose(acceptSocket); - try { - acceptSocket.close(); - } - catch (IOException e) { - LOG.warn("AcceptTask: close() failed: this={}", this, e); - } - break; - } - else { - eventHandler.onReceive(acceptSocket, len, byteBuf); - lastEventTimeStampMilli.set(System.currentTimeMillis()); - } - } - catch (IOException e) { - LOG.warn("AcceptTask: recv() failed: this={}, message={}, cause={}", - this, e.getMessage(), e.getCause() == null ? "" : e.getCause().getMessage()); - if (e instanceof SSLHandshakeException && e.getCause() instanceof EOFException) { - eventHandler.onClose(acceptSocket); - break; - } - if (acceptSocket.isClosed()) { - eventHandler.onClose(acceptSocket); - throw new RuntimeException(e); - } - } - } - } - finally { - try { - LOG.debug("AcceptTask: Finished. Closing... this={}, local={}, remote={}", - this, acceptSocket.getLocalPort(), acceptSocket.getPort()); - acceptSocket.close(); - } - catch (IOException e) { - LOG.warn("AcceptTask: close() failed", e); - } + acceptSocket.close(); + } catch (IOException e) { + LOG.warn("AcceptTask: close() failed: this={}", this, e); } + break; + } else { + eventHandler.onReceive(acceptSocket, len, byteBuf); + lastEventTimeStampMilli.set(System.currentTimeMillis()); + } + } catch (IOException e) { + LOG.warn( + "AcceptTask: recv() failed: this={}, message={}, cause={}", + this, + e.getMessage(), + e.getCause() == null ? "" : e.getCause().getMessage()); + if (e instanceof SSLHandshakeException && e.getCause() instanceof EOFException) { + eventHandler.onClose(acceptSocket); + break; + } + if (acceptSocket.isClosed()) { + eventHandler.onClose(acceptSocket); + throw new RuntimeException(e); + } } + } + } finally { + try { + LOG.debug( + "AcceptTask: Finished. Closing... this={}, local={}, remote={}", + this, + acceptSocket.getLocalPort(), + acceptSocket.getPort()); + acceptSocket.close(); + } catch (IOException e) { + LOG.warn("AcceptTask: close() failed", e); + } } + } } + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/MockTCPServerWithMetrics.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/MockTCPServerWithMetrics.java index 5f7d9c20..b18d299f 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/MockTCPServerWithMetrics.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/MockTCPServerWithMetrics.java @@ -16,55 +16,48 @@ package org.komamitsu.fluency.fluentd; -import org.komamitsu.fluency.util.Tuple; - import java.net.Socket; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; +import org.komamitsu.fluency.util.Tuple; -public class MockTCPServerWithMetrics - extends MockTCPServer -{ - private final List> events = new CopyOnWriteArrayList>(); - private final EventHandler eventHandler = new EventHandler() - { +public class MockTCPServerWithMetrics extends MockTCPServer { + private final List> events = + new CopyOnWriteArrayList>(); + private final EventHandler eventHandler = + new EventHandler() { @Override - public void onConnect(Socket acceptSocket) - { - events.add(new Tuple(Type.CONNECT, null)); + public void onConnect(Socket acceptSocket) { + events.add(new Tuple(Type.CONNECT, null)); } @Override - public void onReceive(Socket acceptSocket, int len, byte[] data) - { - events.add(new Tuple(Type.RECEIVE, len)); + public void onReceive(Socket acceptSocket, int len, byte[] data) { + events.add(new Tuple(Type.RECEIVE, len)); } @Override - public void onClose(Socket acceptSocket) - { - events.add(new Tuple(Type.CLOSE, null)); + public void onClose(Socket acceptSocket) { + events.add(new Tuple(Type.CLOSE, null)); } - }; - - public MockTCPServerWithMetrics(boolean sslEnabled) - { - super(sslEnabled); - } - - @Override - protected EventHandler getEventHandler() - { - return eventHandler; - } - - public List> getEvents() - { - return events; - } - - public enum Type - { - CONNECT, RECEIVE, CLOSE; - } + }; + + public MockTCPServerWithMetrics(boolean sslEnabled) { + super(sslEnabled); + } + + @Override + protected EventHandler getEventHandler() { + return eventHandler; + } + + public List> getEvents() { + return events; + } + + public enum Type { + CONNECT, + RECEIVE, + CLOSE; + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/SSLTestSocketFactories.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/SSLTestSocketFactories.java index f341caf1..54a8504f 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/SSLTestSocketFactories.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/SSLTestSocketFactories.java @@ -16,70 +16,76 @@ package org.komamitsu.fluency.fluentd; -import org.komamitsu.fluency.fluentd.ingester.sender.SSLSocketBuilder; - -import javax.net.ssl.*; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.net.InetSocketAddress; import java.security.*; import java.security.cert.CertificateException; +import javax.net.ssl.*; +import org.komamitsu.fluency.fluentd.ingester.sender.SSLSocketBuilder; -public class SSLTestSocketFactories -{ - private static final String KEYSTORE_PASSWORD = "keypassword"; - private static final String KEY_PASSWORD = "keypassword"; - private static final String TRUSTSTORE_PASSWORD = "trustpassword"; +public class SSLTestSocketFactories { + private static final String KEYSTORE_PASSWORD = "keypassword"; + private static final String KEY_PASSWORD = "keypassword"; + private static final String TRUSTSTORE_PASSWORD = "trustpassword"; - public static final SSLSocketFactory SSL_CLIENT_SOCKET_FACTORY = createClientSocketFactory(); + public static final SSLSocketFactory SSL_CLIENT_SOCKET_FACTORY = createClientSocketFactory(); - private static SSLSocketFactory createClientSocketFactory() - { - String trustStorePath = SSLSocketBuilder.class.getClassLoader().getResource("truststore.jks").getFile(); + private static SSLSocketFactory createClientSocketFactory() { + String trustStorePath = + SSLSocketBuilder.class.getClassLoader().getResource("truststore.jks").getFile(); - try (InputStream keystoreStream = new FileInputStream(trustStorePath)) { - KeyStore keystore = KeyStore.getInstance("JKS"); + try (InputStream keystoreStream = new FileInputStream(trustStorePath)) { + KeyStore keystore = KeyStore.getInstance("JKS"); - keystore.load(keystoreStream, TRUSTSTORE_PASSWORD.toCharArray()); - TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + keystore.load(keystoreStream, TRUSTSTORE_PASSWORD.toCharArray()); + TrustManagerFactory trustManagerFactory = + TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - trustManagerFactory.init(keystore); + trustManagerFactory.init(keystore); - SSLContext sslContext = SSLContext.getInstance("TLSv1.3"); - sslContext.init(null, trustManagerFactory.getTrustManagers(), new SecureRandom()); + SSLContext sslContext = SSLContext.getInstance("TLSv1.3"); + sslContext.init(null, trustManagerFactory.getTrustManagers(), new SecureRandom()); - return sslContext.getSocketFactory(); - } - catch (Throwable e) { - throw new RuntimeException("Failed to create SSLSocketFactory", e); - } + return sslContext.getSocketFactory(); + } catch (Throwable e) { + throw new RuntimeException("Failed to create SSLSocketFactory", e); } + } - public static SSLServerSocket createServerSocket() - throws IOException, CertificateException, NoSuchAlgorithmException, UnrecoverableKeyException, KeyStoreException, KeyManagementException - { - String trustStorePath = SSLSocketBuilder.class.getClassLoader().getResource("truststore.jks").getFile(); - System.getProperties().setProperty("javax.net.ssl.trustStore", trustStorePath); + public static SSLServerSocket createServerSocket() + throws IOException, + CertificateException, + NoSuchAlgorithmException, + UnrecoverableKeyException, + KeyStoreException, + KeyManagementException { + String trustStorePath = + SSLSocketBuilder.class.getClassLoader().getResource("truststore.jks").getFile(); + System.getProperties().setProperty("javax.net.ssl.trustStore", trustStorePath); - String keyStorePath = SSLSocketBuilder.class.getClassLoader().getResource("keystore.jks").getFile(); + String keyStorePath = + SSLSocketBuilder.class.getClassLoader().getResource("keystore.jks").getFile(); - try (InputStream keystoreStream = new FileInputStream(keyStorePath)) { - KeyStore keystore = KeyStore.getInstance("JKS"); + try (InputStream keystoreStream = new FileInputStream(keyStorePath)) { + KeyStore keystore = KeyStore.getInstance("JKS"); - keystore.load(keystoreStream, KEYSTORE_PASSWORD.toCharArray()); - KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keystore.load(keystoreStream, KEYSTORE_PASSWORD.toCharArray()); + KeyManagerFactory keyManagerFactory = + KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); - keyManagerFactory.init(keystore, KEY_PASSWORD.toCharArray()); + keyManagerFactory.init(keystore, KEY_PASSWORD.toCharArray()); - SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); - sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom()); + SSLContext sslContext = SSLContext.getInstance("TLSv1.2"); + sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom()); - SSLServerSocket serverSocket = (SSLServerSocket) sslContext.getServerSocketFactory().createServerSocket(); - serverSocket.setEnabledCipherSuites(serverSocket.getSupportedCipherSuites()); - serverSocket.bind(new InetSocketAddress(0)); + SSLServerSocket serverSocket = + (SSLServerSocket) sslContext.getServerSocketFactory().createServerSocket(); + serverSocket.setEnabledCipherSuites(serverSocket.getSupportedCipherSuites()); + serverSocket.bind(new InetSocketAddress(0)); - return serverSocket; - } + return serverSocket; } -} \ No newline at end of file + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/FluentdIngesterTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/FluentdIngesterTest.java index 7926ce56..a1cac1d2 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/FluentdIngesterTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/FluentdIngesterTest.java @@ -16,6 +16,18 @@ package org.komamitsu.fluency.fluentd.ingester; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.komamitsu.fluency.fluentd.ingester.sender.FluentdSender; @@ -29,110 +41,85 @@ import org.msgpack.value.Value; import org.msgpack.value.ValueFactory; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -class FluentdIngesterTest -{ - private static final String TAG = "foo.bar"; - private static final byte[] DATA = "hello, world".getBytes(StandardCharsets.UTF_8); - @Captor - public ArgumentCaptor> byteBuffersArgumentCaptor; - private FluentdSender fluentdSender; - - @BeforeEach - void setUp() - throws Exception - { - MockitoAnnotations.initMocks(this); - fluentdSender = mock(FluentdSender.class); - } - - private byte[] getIngestedData(List byteBuffers) - throws IOException - { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - for (ByteBuffer byteBuffer : byteBuffers) { - outputStream.write(byteBuffer.array()); - } - return outputStream.toByteArray(); - } - - @Test - void ingestWithoutAck() - throws IOException - { - Ingester ingester = new FluentdIngester(new FluentdIngester.Config(), fluentdSender); - ingester.ingest(TAG, ByteBuffer.wrap(DATA)); - - verify(fluentdSender, times(1)).send(byteBuffersArgumentCaptor.capture()); - List byteBuffers = byteBuffersArgumentCaptor.getAllValues().get(0); - byte[] ingested = getIngestedData(byteBuffers); - - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(ingested); - ImmutableArrayValue arrayValue = unpacker.unpackValue().asArrayValue(); - assertEquals(3, arrayValue.size()); - assertEquals(TAG, arrayValue.get(0).asStringValue().asString()); - assertArrayEquals(DATA, arrayValue.get(1).asRawValue().asByteArray()); - Map options = arrayValue.get(2).asMapValue().map(); - assertEquals(1, options.size()); - assertEquals(DATA.length, - options.get(ValueFactory.newString("size")).asIntegerValue().asInt()); - } - - @Test - void ingestWithAck() - throws IOException - { - FluentdIngester.Config config = new FluentdIngester.Config(); - config.setAckResponseMode(true); - Ingester ingester = new FluentdIngester(config, fluentdSender); - ingester.ingest(TAG, ByteBuffer.wrap(DATA)); - - ArgumentCaptor ackTokenArgumentCaptor = ArgumentCaptor.forClass(String.class); - verify(fluentdSender, times(1)) - .sendWithAck(byteBuffersArgumentCaptor.capture(), ackTokenArgumentCaptor.capture()); - List byteBuffers = byteBuffersArgumentCaptor.getAllValues().get(0); - byte[] ingested = getIngestedData(byteBuffers); - - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(ingested); - ImmutableArrayValue arrayValue = unpacker.unpackValue().asArrayValue(); - assertEquals(3, arrayValue.size()); - assertEquals(TAG, arrayValue.get(0).asStringValue().asString()); - assertArrayEquals(DATA, arrayValue.get(1).asRawValue().asByteArray()); - Map options = arrayValue.get(2).asMapValue().map(); - assertEquals(2, options.size()); - assertEquals(DATA.length, options.get(ValueFactory.newString("size")).asIntegerValue().asInt()); - String ackToken = options.get(ValueFactory.newString("chunk")).asRawValue().asString(); - - List ackTokenArgumentCaptorAllValues = ackTokenArgumentCaptor.getAllValues(); - assertEquals(1, ackTokenArgumentCaptorAllValues.size()); - assertEquals(ackToken, ackTokenArgumentCaptorAllValues.get(0)); - } - - @Test - void getSender() - { - assertEquals(fluentdSender, new FluentdIngester(new FluentdIngester.Config(), fluentdSender).getSender()); - } - - @Test - void close() - throws IOException - { - Ingester ingester = new FluentdIngester(new FluentdIngester.Config(), fluentdSender); - ingester.close(); - - verify(fluentdSender, times(1)).close(); +class FluentdIngesterTest { + private static final String TAG = "foo.bar"; + private static final byte[] DATA = "hello, world".getBytes(StandardCharsets.UTF_8); + @Captor public ArgumentCaptor> byteBuffersArgumentCaptor; + private FluentdSender fluentdSender; + + @BeforeEach + void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + fluentdSender = mock(FluentdSender.class); + } + + private byte[] getIngestedData(List byteBuffers) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + for (ByteBuffer byteBuffer : byteBuffers) { + outputStream.write(byteBuffer.array()); } + return outputStream.toByteArray(); + } + + @Test + void ingestWithoutAck() throws IOException { + Ingester ingester = new FluentdIngester(new FluentdIngester.Config(), fluentdSender); + ingester.ingest(TAG, ByteBuffer.wrap(DATA)); + + verify(fluentdSender, times(1)).send(byteBuffersArgumentCaptor.capture()); + List byteBuffers = byteBuffersArgumentCaptor.getAllValues().get(0); + byte[] ingested = getIngestedData(byteBuffers); + + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(ingested); + ImmutableArrayValue arrayValue = unpacker.unpackValue().asArrayValue(); + assertEquals(3, arrayValue.size()); + assertEquals(TAG, arrayValue.get(0).asStringValue().asString()); + assertArrayEquals(DATA, arrayValue.get(1).asRawValue().asByteArray()); + Map options = arrayValue.get(2).asMapValue().map(); + assertEquals(1, options.size()); + assertEquals(DATA.length, options.get(ValueFactory.newString("size")).asIntegerValue().asInt()); + } + + @Test + void ingestWithAck() throws IOException { + FluentdIngester.Config config = new FluentdIngester.Config(); + config.setAckResponseMode(true); + Ingester ingester = new FluentdIngester(config, fluentdSender); + ingester.ingest(TAG, ByteBuffer.wrap(DATA)); + + ArgumentCaptor ackTokenArgumentCaptor = ArgumentCaptor.forClass(String.class); + verify(fluentdSender, times(1)) + .sendWithAck(byteBuffersArgumentCaptor.capture(), ackTokenArgumentCaptor.capture()); + List byteBuffers = byteBuffersArgumentCaptor.getAllValues().get(0); + byte[] ingested = getIngestedData(byteBuffers); + + MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(ingested); + ImmutableArrayValue arrayValue = unpacker.unpackValue().asArrayValue(); + assertEquals(3, arrayValue.size()); + assertEquals(TAG, arrayValue.get(0).asStringValue().asString()); + assertArrayEquals(DATA, arrayValue.get(1).asRawValue().asByteArray()); + Map options = arrayValue.get(2).asMapValue().map(); + assertEquals(2, options.size()); + assertEquals(DATA.length, options.get(ValueFactory.newString("size")).asIntegerValue().asInt()); + String ackToken = options.get(ValueFactory.newString("chunk")).asRawValue().asString(); + + List ackTokenArgumentCaptorAllValues = ackTokenArgumentCaptor.getAllValues(); + assertEquals(1, ackTokenArgumentCaptorAllValues.size()); + assertEquals(ackToken, ackTokenArgumentCaptorAllValues.get(0)); + } + + @Test + void getSender() { + assertEquals( + fluentdSender, + new FluentdIngester(new FluentdIngester.Config(), fluentdSender).getSender()); + } + + @Test + void close() throws IOException { + Ingester ingester = new FluentdIngester(new FluentdIngester.Config(), fluentdSender); + ingester.close(); + + verify(fluentdSender, times(1)).close(); + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/ConfigTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/ConfigTest.java index 3f152e86..57ae7657 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/ConfigTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/ConfigTest.java @@ -16,68 +16,54 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.fail; - -class ConfigTest -{ - @Test - void errorHandler() - throws IOException - { - final AtomicBoolean errorOccurred = new AtomicBoolean(); - FluentdSender.Config config = new FluentdSender.Config(); - config.setErrorHandler(e -> errorOccurred.set(true)); +class ConfigTest { + @Test + void errorHandler() throws IOException { + final AtomicBoolean errorOccurred = new AtomicBoolean(); + FluentdSender.Config config = new FluentdSender.Config(); + config.setErrorHandler(e -> errorOccurred.set(true)); - new DummySender(config, false).send(ByteBuffer.allocate(8)); - assertThat(errorOccurred.get()).isEqualTo(false); + new DummySender(config, false).send(ByteBuffer.allocate(8)); + assertThat(errorOccurred.get()).isEqualTo(false); - try { - new DummySender(config, true).send(ByteBuffer.allocate(8)); - fail(); - } - catch (Exception e) { - assertThat(errorOccurred.get()).isEqualTo(true); - } + try { + new DummySender(config, true).send(ByteBuffer.allocate(8)); + fail(); + } catch (Exception e) { + assertThat(errorOccurred.get()).isEqualTo(true); } + } - static class DummySender - extends FluentdSender - { - private final boolean shouldFail; - - protected DummySender(Config config, boolean shouldFail) - { - super(config); - this.shouldFail = shouldFail; - } + static class DummySender extends FluentdSender { + private final boolean shouldFail; - @Override - public boolean isAvailable() - { - return true; - } + protected DummySender(Config config, boolean shouldFail) { + super(config); + this.shouldFail = shouldFail; + } - @Override - protected void sendInternal(List buffers, String ackToken) - throws IOException - { - if (shouldFail) { - throw new RuntimeException("Unexpected scheduled error occurred"); - } - } + @Override + public boolean isAvailable() { + return true; + } - @Override - public void close() - throws IOException - { - } + @Override + protected void sendInternal(List buffers, String ackToken) throws IOException { + if (shouldFail) { + throw new RuntimeException("Unexpected scheduled error occurred"); + } } + + @Override + public void close() throws IOException {} + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MockMultiTCPServerWithMetrics.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MockMultiTCPServerWithMetrics.java index c250a610..573c6175 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MockMultiTCPServerWithMetrics.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MockMultiTCPServerWithMetrics.java @@ -16,9 +16,7 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.komamitsu.fluency.fluentd.MockTCPServerWithMetrics; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.net.InetSocketAddress; @@ -29,77 +27,66 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import org.komamitsu.fluency.fluentd.MockTCPServerWithMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class MockMultiTCPServerWithMetrics - extends MockTCPServerWithMetrics -{ - private static final Logger LOG = LoggerFactory.getLogger(MockMultiTCPServerWithMetrics.class); - private ExecutorService executorService; - private DatagramChannel channel; +public class MockMultiTCPServerWithMetrics extends MockTCPServerWithMetrics { + private static final Logger LOG = LoggerFactory.getLogger(MockMultiTCPServerWithMetrics.class); + private ExecutorService executorService; + private DatagramChannel channel; - MockMultiTCPServerWithMetrics(boolean sslEnabled) - throws IOException - { - super(sslEnabled); - } + MockMultiTCPServerWithMetrics(boolean sslEnabled) throws IOException { + super(sslEnabled); + } - @Override - public synchronized void start() - throws Exception - { - super.start(); + @Override + public synchronized void start() throws Exception { + super.start(); - if (executorService != null) { - return; - } + if (executorService != null) { + return; + } - executorService = Executors.newSingleThreadExecutor(); - channel = DatagramChannel.open(); - channel.socket().bind(new InetSocketAddress(getLocalPort())); + executorService = Executors.newSingleThreadExecutor(); + channel = DatagramChannel.open(); + channel.socket().bind(new InetSocketAddress(getLocalPort())); - executorService.execute(new Runnable() - { - @Override - public void run() - { - while (executorService != null && !executorService.isTerminated()) { - try { - ByteBuffer buffer = ByteBuffer.allocate(8); - SocketAddress socketAddress = channel.receive(buffer); - assertEquals(0, buffer.position()); - channel.send(buffer, socketAddress); - } - catch (ClosedByInterruptException e) { - // Expected - } - catch (IOException e) { - LOG.warn("Failed to receive or send heartbeat", e); - } - } + executorService.execute( + new Runnable() { + @Override + public void run() { + while (executorService != null && !executorService.isTerminated()) { + try { + ByteBuffer buffer = ByteBuffer.allocate(8); + SocketAddress socketAddress = channel.receive(buffer); + assertEquals(0, buffer.position()); + channel.send(buffer, socketAddress); + } catch (ClosedByInterruptException e) { + // Expected + } catch (IOException e) { + LOG.warn("Failed to receive or send heartbeat", e); + } } + } }); - } + } - @Override - public synchronized void stop() - throws IOException - { - super.stop(); + @Override + public synchronized void stop() throws IOException { + super.stop(); - if (executorService == null) { - return; - } + if (executorService == null) { + return; + } - executorService.shutdown(); - try { - executorService.awaitTermination(2, TimeUnit.SECONDS); - } - catch (InterruptedException e) { - LOG.warn("stop(): Interrupted while shutting down", e); - } - executorService.shutdownNow(); - executorService = null; + executorService.shutdown(); + try { + executorService.awaitTermination(2, TimeUnit.SECONDS); + } catch (InterruptedException e) { + LOG.warn("stop(): Interrupted while shutting down", e); } + executorService.shutdownNow(); + executorService = null; + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MockTCPSender.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MockTCPSender.java index aa8dba68..a458c2e6 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MockTCPSender.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MockTCPSender.java @@ -16,101 +16,89 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class MockTCPSender - extends TCPSender -{ - private static final Logger LOG = LoggerFactory.getLogger(MockTCPSender.class); - - private final List events = new ArrayList(); - private final AtomicInteger closeCount = new AtomicInteger(); - - public MockTCPSender(TCPSender.Config config) - throws IOException - { - super(config); +public class MockTCPSender extends TCPSender { + private static final Logger LOG = LoggerFactory.getLogger(MockTCPSender.class); + + private final List events = new ArrayList(); + private final AtomicInteger closeCount = new AtomicInteger(); + + public MockTCPSender(TCPSender.Config config) throws IOException { + super(config); + } + + @Override + protected synchronized void sendInternal(List buffers, String ackToken) + throws IOException { + assertEquals(3, buffers.size()); + Event event = new Event(); + int i = 0; + for (ByteBuffer data : buffers) { + ByteBuffer byteBuffer = ByteBuffer.allocateDirect(data.capacity()); + byteBuffer.put(data); + byteBuffer.flip(); + switch (i) { + case 0: + event.header = byteBuffer; + break; + case 1: + event.data = byteBuffer; + break; + case 2: + event.option = byteBuffer; + break; + default: + throw new IllegalStateException("Shouldn't reach here"); + } + i++; } + events.add(event); + } - @Override - protected synchronized void sendInternal(List buffers, String ackToken) - throws IOException - { - assertEquals(3, buffers.size()); - Event event = new Event(); - int i = 0; - for (ByteBuffer data : buffers) { - ByteBuffer byteBuffer = ByteBuffer.allocateDirect(data.capacity()); - byteBuffer.put(data); - byteBuffer.flip(); - switch (i) { - case 0: - event.header = byteBuffer; - break; - case 1: - event.data = byteBuffer; - break; - case 2: - event.option = byteBuffer; - break; - default: - throw new IllegalStateException("Shouldn't reach here"); - } - i++; - } - events.add(event); - } + @Override + public synchronized void close() throws IOException { + closeCount.incrementAndGet(); + } - @Override - public synchronized void close() - throws IOException - { - closeCount.incrementAndGet(); - } + public List getEvents() { + return events; + } - public List getEvents() - { - return events; - } - - public AtomicInteger getCloseCount() - { - return closeCount; - } + public AtomicInteger getCloseCount() { + return closeCount; + } - public static class Event - { - ByteBuffer header; - ByteBuffer data; - ByteBuffer option; + public static class Event { + ByteBuffer header; + ByteBuffer data; + ByteBuffer option; - public byte[] getAllBytes() - { - byte[] bytes = new byte[header.limit() + data.limit() + option.limit()]; + public byte[] getAllBytes() { + byte[] bytes = new byte[header.limit() + data.limit() + option.limit()]; - int pos = 0; - int len = header.limit(); - header.get(bytes, pos, len); - pos += len; + int pos = 0; + int len = header.limit(); + header.get(bytes, pos, len); + pos += len; - len = data.limit(); - data.get(bytes, pos, len); - pos += len; + len = data.limit(); + data.get(bytes, pos, len); + pos += len; - len = option.limit(); - option.get(bytes, pos, len); - pos += len; + len = option.limit(); + option.get(bytes, pos, len); + pos += len; - return bytes; - } + return bytes; } + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MultiSenderTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MultiSenderTest.java index 3f5169c4..02a4a2a2 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MultiSenderTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/MultiSenderTest.java @@ -16,16 +16,11 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.fluentd.MockTCPServerWithMetrics; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; -import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.*; -import org.komamitsu.fluency.util.Tuple; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; +import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; +import com.google.common.collect.ImmutableList; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.Charset; @@ -35,283 +30,284 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.fluentd.MockTCPServerWithMetrics; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; +import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.*; +import org.komamitsu.fluency.util.Tuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.*; -import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; - -class MultiSenderTest -{ - private static final Logger LOG = LoggerFactory.getLogger(MultiSenderTest.class); - - private FailureDetector createFailureDetector(Heartbeater hb) - { - return new FailureDetector(new PhiAccrualFailureDetectStrategy(), hb); - } - - @Test - void testConstructorForTCPSender() - throws IOException - { - MultiSender multiSender = null; - try { - TCPSender.Config senderConfig0 = new TCPSender.Config(); - senderConfig0.setPort(24225); - - TCPHeartbeater.Config hbConfig0 = new TCPHeartbeater.Config(); - hbConfig0.setPort(24225); - - TCPSender.Config senderConfig1 = new TCPSender.Config(); - senderConfig1.setHost("0.0.0.0"); - senderConfig1.setPort(24226); - - TCPHeartbeater.Config hbConfig1 = new TCPHeartbeater.Config(); - hbConfig1.setHost("0.0.0.0"); - hbConfig1.setPort(24226); - - multiSender = new MultiSender(new MultiSender.Config(), - Arrays.asList( - new TCPSender(senderConfig0, - createFailureDetector(new TCPHeartbeater(hbConfig0))), - new TCPSender(senderConfig1, - createFailureDetector(new TCPHeartbeater(hbConfig1))))); - - assertThat(multiSender.toString().length()).isGreaterThan(0); - - assertEquals(2, multiSender.getSenders().size()); - - TCPSender tcpSender = (TCPSender) multiSender.getSenders().get(0); - assertEquals("127.0.0.1", tcpSender.getHost()); - assertEquals(24225, tcpSender.getPort()); - assertTrue(tcpSender.getFailureDetector().getHeartbeater() instanceof InetSocketHeartbeater); - { - InetSocketHeartbeater hb = (InetSocketHeartbeater) tcpSender.getFailureDetector().getHeartbeater(); - assertEquals("127.0.0.1", hb.getHost()); - assertEquals(24225, hb.getPort()); - } - - tcpSender = (TCPSender) multiSender.getSenders().get(1); - assertEquals("0.0.0.0", tcpSender.getHost()); - assertEquals(24226, tcpSender.getPort()); - assertTrue(tcpSender.getFailureDetector().getHeartbeater() instanceof InetSocketHeartbeater); - { - InetSocketHeartbeater hb = (InetSocketHeartbeater) tcpSender.getFailureDetector().getHeartbeater(); - assertEquals("0.0.0.0", hb.getHost()); - assertEquals(24226, hb.getPort()); - } - } - finally { - if (multiSender != null) { - multiSender.close(); - } - } - } - - @Test - void testConstructorForSSLSender() - throws IOException - { - MultiSender multiSender = null; - try { - - SSLSender.Config senderConfig0 = new SSLSender.Config(); - senderConfig0.setPort(24225); - senderConfig0.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - - SSLHeartbeater.Config hbConfig0 = new SSLHeartbeater.Config(); - hbConfig0.setPort(24225); - - SSLSender.Config senderConfig1 = new SSLSender.Config(); - senderConfig1.setHost("0.0.0.0"); - senderConfig1.setPort(24226); - senderConfig1.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - - SSLHeartbeater.Config hbConfig1 = new SSLHeartbeater.Config(); - hbConfig1.setHost("0.0.0.0"); - hbConfig1.setPort(24226); - - multiSender = new MultiSender(new MultiSender.Config(), - Arrays.asList( - new SSLSender(senderConfig0, - createFailureDetector(new SSLHeartbeater(hbConfig0))), - new SSLSender(senderConfig1, - createFailureDetector(new SSLHeartbeater(hbConfig1))))); - - assertThat(multiSender.toString().length()).isGreaterThan(0); - - assertEquals(2, multiSender.getSenders().size()); - - SSLSender sslSender = (SSLSender) multiSender.getSenders().get(0); - assertEquals("127.0.0.1", sslSender.getHost()); - assertEquals(24225, sslSender.getPort()); - assertTrue(sslSender.getFailureDetector().getHeartbeater() instanceof InetSocketHeartbeater); - { - InetSocketHeartbeater hb = (InetSocketHeartbeater) sslSender.getFailureDetector().getHeartbeater(); - assertEquals("127.0.0.1", hb.getHost()); - assertEquals(24225, hb.getPort()); - } - - sslSender = (SSLSender) multiSender.getSenders().get(1); - assertEquals("0.0.0.0", sslSender.getHost()); - assertEquals(24226, sslSender.getPort()); - assertTrue(sslSender.getFailureDetector().getHeartbeater() instanceof InetSocketHeartbeater); - { - InetSocketHeartbeater hb = (InetSocketHeartbeater) sslSender.getFailureDetector().getHeartbeater(); - assertEquals("0.0.0.0", hb.getHost()); - assertEquals(24226, hb.getPort()); - } - } - finally { - if (multiSender != null) { - multiSender.close(); - } - } +class MultiSenderTest { + private static final Logger LOG = LoggerFactory.getLogger(MultiSenderTest.class); + + private FailureDetector createFailureDetector(Heartbeater hb) { + return new FailureDetector(new PhiAccrualFailureDetectStrategy(), hb); + } + + @Test + void testConstructorForTCPSender() throws IOException { + MultiSender multiSender = null; + try { + TCPSender.Config senderConfig0 = new TCPSender.Config(); + senderConfig0.setPort(24225); + + TCPHeartbeater.Config hbConfig0 = new TCPHeartbeater.Config(); + hbConfig0.setPort(24225); + + TCPSender.Config senderConfig1 = new TCPSender.Config(); + senderConfig1.setHost("0.0.0.0"); + senderConfig1.setPort(24226); + + TCPHeartbeater.Config hbConfig1 = new TCPHeartbeater.Config(); + hbConfig1.setHost("0.0.0.0"); + hbConfig1.setPort(24226); + + multiSender = + new MultiSender( + new MultiSender.Config(), + Arrays.asList( + new TCPSender( + senderConfig0, createFailureDetector(new TCPHeartbeater(hbConfig0))), + new TCPSender( + senderConfig1, createFailureDetector(new TCPHeartbeater(hbConfig1))))); + + assertThat(multiSender.toString().length()).isGreaterThan(0); + + assertEquals(2, multiSender.getSenders().size()); + + TCPSender tcpSender = (TCPSender) multiSender.getSenders().get(0); + assertEquals("127.0.0.1", tcpSender.getHost()); + assertEquals(24225, tcpSender.getPort()); + assertTrue(tcpSender.getFailureDetector().getHeartbeater() instanceof InetSocketHeartbeater); + { + InetSocketHeartbeater hb = + (InetSocketHeartbeater) tcpSender.getFailureDetector().getHeartbeater(); + assertEquals("127.0.0.1", hb.getHost()); + assertEquals(24225, hb.getPort()); + } + + tcpSender = (TCPSender) multiSender.getSenders().get(1); + assertEquals("0.0.0.0", tcpSender.getHost()); + assertEquals(24226, tcpSender.getPort()); + assertTrue(tcpSender.getFailureDetector().getHeartbeater() instanceof InetSocketHeartbeater); + { + InetSocketHeartbeater hb = + (InetSocketHeartbeater) tcpSender.getFailureDetector().getHeartbeater(); + assertEquals("0.0.0.0", hb.getHost()); + assertEquals(24226, hb.getPort()); + } + } finally { + if (multiSender != null) { + multiSender.close(); + } } - - @Test - void testTCPSend() - throws Exception - { - testSend(false); + } + + @Test + void testConstructorForSSLSender() throws IOException { + MultiSender multiSender = null; + try { + + SSLSender.Config senderConfig0 = new SSLSender.Config(); + senderConfig0.setPort(24225); + senderConfig0.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + + SSLHeartbeater.Config hbConfig0 = new SSLHeartbeater.Config(); + hbConfig0.setPort(24225); + + SSLSender.Config senderConfig1 = new SSLSender.Config(); + senderConfig1.setHost("0.0.0.0"); + senderConfig1.setPort(24226); + senderConfig1.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + + SSLHeartbeater.Config hbConfig1 = new SSLHeartbeater.Config(); + hbConfig1.setHost("0.0.0.0"); + hbConfig1.setPort(24226); + + multiSender = + new MultiSender( + new MultiSender.Config(), + Arrays.asList( + new SSLSender( + senderConfig0, createFailureDetector(new SSLHeartbeater(hbConfig0))), + new SSLSender( + senderConfig1, createFailureDetector(new SSLHeartbeater(hbConfig1))))); + + assertThat(multiSender.toString().length()).isGreaterThan(0); + + assertEquals(2, multiSender.getSenders().size()); + + SSLSender sslSender = (SSLSender) multiSender.getSenders().get(0); + assertEquals("127.0.0.1", sslSender.getHost()); + assertEquals(24225, sslSender.getPort()); + assertTrue(sslSender.getFailureDetector().getHeartbeater() instanceof InetSocketHeartbeater); + { + InetSocketHeartbeater hb = + (InetSocketHeartbeater) sslSender.getFailureDetector().getHeartbeater(); + assertEquals("127.0.0.1", hb.getHost()); + assertEquals(24225, hb.getPort()); + } + + sslSender = (SSLSender) multiSender.getSenders().get(1); + assertEquals("0.0.0.0", sslSender.getHost()); + assertEquals(24226, sslSender.getPort()); + assertTrue(sslSender.getFailureDetector().getHeartbeater() instanceof InetSocketHeartbeater); + { + InetSocketHeartbeater hb = + (InetSocketHeartbeater) sslSender.getFailureDetector().getHeartbeater(); + assertEquals("0.0.0.0", hb.getHost()); + assertEquals(24226, hb.getPort()); + } + } finally { + if (multiSender != null) { + multiSender.close(); + } } - - @Test - void testSSLSend() - throws Exception - { - testSend(true); + } + + @Test + void testTCPSend() throws Exception { + testSend(false); + } + + @Test + void testSSLSend() throws Exception { + testSend(true); + } + + private void testSend(boolean sslEnabled) throws Exception { + final MockMultiTCPServerWithMetrics server0 = new MockMultiTCPServerWithMetrics(sslEnabled); + server0.start(); + final MockMultiTCPServerWithMetrics server1 = new MockMultiTCPServerWithMetrics(sslEnabled); + server1.start(); + + int concurency = 20; + final int reqNum = 5000; + final CountDownLatch latch = new CountDownLatch(concurency); + + final MultiSender sender; + + if (sslEnabled) { + SSLSender.Config senderConfig0 = new SSLSender.Config(); + senderConfig0.setPort(server0.getLocalPort()); + senderConfig0.setReadTimeoutMilli(500); + senderConfig0.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + + UDPHeartbeater.Config hbConfig0 = new UDPHeartbeater.Config(); + hbConfig0.setPort(server0.getLocalPort()); + + SSLSender.Config senderConfig1 = new SSLSender.Config(); + senderConfig1.setPort(server1.getLocalPort()); + senderConfig1.setReadTimeoutMilli(500); + senderConfig1.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + + UDPHeartbeater.Config hbConfig1 = new UDPHeartbeater.Config(); + hbConfig1.setPort(server1.getLocalPort()); + + sender = + new MultiSender( + new MultiSender.Config(), + ImmutableList.of( + new SSLSender( + senderConfig0, createFailureDetector(new UDPHeartbeater(hbConfig0))), + new SSLSender( + senderConfig1, createFailureDetector(new UDPHeartbeater(hbConfig1))))); + } else { + TCPSender.Config senderConfig0 = new TCPSender.Config(); + senderConfig0.setPort(server0.getLocalPort()); + senderConfig0.setReadTimeoutMilli(500); + + UDPHeartbeater.Config hbConfig0 = new UDPHeartbeater.Config(); + hbConfig0.setPort(server0.getLocalPort()); + + TCPSender.Config senderConfig1 = new TCPSender.Config(); + senderConfig1.setPort(server1.getLocalPort()); + senderConfig1.setReadTimeoutMilli(500); + + UDPHeartbeater.Config hbConfig1 = new UDPHeartbeater.Config(); + hbConfig1.setPort(server0.getLocalPort()); + + sender = + new MultiSender( + new MultiSender.Config(), + ImmutableList.of( + new TCPSender( + senderConfig0, createFailureDetector(new UDPHeartbeater(hbConfig0))), + new TCPSender( + senderConfig1, createFailureDetector(new UDPHeartbeater(hbConfig0))))); } - private void testSend(boolean sslEnabled) - throws Exception - { - final MockMultiTCPServerWithMetrics server0 = new MockMultiTCPServerWithMetrics(sslEnabled); - server0.start(); - final MockMultiTCPServerWithMetrics server1 = new MockMultiTCPServerWithMetrics(sslEnabled); - server1.start(); - - int concurency = 20; - final int reqNum = 5000; - final CountDownLatch latch = new CountDownLatch(concurency); - - final MultiSender sender; - - if (sslEnabled) { - SSLSender.Config senderConfig0 = new SSLSender.Config(); - senderConfig0.setPort(server0.getLocalPort()); - senderConfig0.setReadTimeoutMilli(500); - senderConfig0.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - - UDPHeartbeater.Config hbConfig0 = new UDPHeartbeater.Config(); - hbConfig0.setPort(server0.getLocalPort()); - - SSLSender.Config senderConfig1 = new SSLSender.Config(); - senderConfig1.setPort(server1.getLocalPort()); - senderConfig1.setReadTimeoutMilli(500); - senderConfig1.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - - UDPHeartbeater.Config hbConfig1 = new UDPHeartbeater.Config(); - hbConfig1.setPort(server1.getLocalPort()); - - sender = new MultiSender(new MultiSender.Config(), - ImmutableList.of( - new SSLSender(senderConfig0, - createFailureDetector(new UDPHeartbeater(hbConfig0))), - new SSLSender(senderConfig1, - createFailureDetector(new UDPHeartbeater(hbConfig1))))); - } - else { - TCPSender.Config senderConfig0 = new TCPSender.Config(); - senderConfig0.setPort(server0.getLocalPort()); - senderConfig0.setReadTimeoutMilli(500); - - UDPHeartbeater.Config hbConfig0 = new UDPHeartbeater.Config(); - hbConfig0.setPort(server0.getLocalPort()); - - TCPSender.Config senderConfig1 = new TCPSender.Config(); - senderConfig1.setPort(server1.getLocalPort()); - senderConfig1.setReadTimeoutMilli(500); - - UDPHeartbeater.Config hbConfig1 = new UDPHeartbeater.Config(); - hbConfig1.setPort(server0.getLocalPort()); - - sender = new MultiSender(new MultiSender.Config(), - ImmutableList.of( - new TCPSender(senderConfig0, - createFailureDetector(new UDPHeartbeater(hbConfig0))), - new TCPSender(senderConfig1, - createFailureDetector(new UDPHeartbeater(hbConfig0))))); - } - - final ExecutorService senderExecutorService = Executors.newCachedThreadPool(); - final AtomicBoolean shouldFailOver = new AtomicBoolean(true); - for (int i = 0; i < concurency; i++) { - senderExecutorService.execute(new Runnable() - { - @Override - public void run() - { - try { - byte[] bytes = "0123456789".getBytes(Charset.forName("UTF-8")); - - for (int j = 0; j < reqNum; j++) { - if (j == reqNum / 2) { - if (shouldFailOver.getAndSet(false)) { - TimeUnit.MILLISECONDS.sleep(100); - LOG.info("Failing over..."); - server0.stop(); - } - } - sender.send(ByteBuffer.wrap(bytes)); - } - latch.countDown(); - } - catch (Exception e) { - LOG.error("Failed to send", e); + final ExecutorService senderExecutorService = Executors.newCachedThreadPool(); + final AtomicBoolean shouldFailOver = new AtomicBoolean(true); + for (int i = 0; i < concurency; i++) { + senderExecutorService.execute( + new Runnable() { + @Override + public void run() { + try { + byte[] bytes = "0123456789".getBytes(Charset.forName("UTF-8")); + + for (int j = 0; j < reqNum; j++) { + if (j == reqNum / 2) { + if (shouldFailOver.getAndSet(false)) { + TimeUnit.MILLISECONDS.sleep(100); + LOG.info("Failing over..."); + server0.stop(); } + } + sender.send(ByteBuffer.wrap(bytes)); } - }); - } + latch.countDown(); + } catch (Exception e) { + LOG.error("Failed to send", e); + } + } + }); + } - if (!latch.await(60, TimeUnit.SECONDS)) { - fail("Sending all requests is timed out"); - } + if (!latch.await(60, TimeUnit.SECONDS)) { + fail("Sending all requests is timed out"); + } - sender.close(); - - server1.waitUntilEventsStop(); - server1.stop(); - - int connectCount = 0; - int closeCount = 0; - long recvCount = 0; - long recvLen = 0; - for (MockMultiTCPServerWithMetrics server : Arrays.asList(server0, server1)) { - for (Tuple event : server.getEvents()) { - switch (event.getFirst()) { - case CONNECT: - connectCount++; - break; - case CLOSE: - closeCount++; - break; - case RECEIVE: - recvCount++; - recvLen += event.getSecond(); - break; - } - } + sender.close(); + + server1.waitUntilEventsStop(); + server1.stop(); + + int connectCount = 0; + int closeCount = 0; + long recvCount = 0; + long recvLen = 0; + for (MockMultiTCPServerWithMetrics server : Arrays.asList(server0, server1)) { + for (Tuple event : server.getEvents()) { + switch (event.getFirst()) { + case CONNECT: + connectCount++; + break; + case CLOSE: + closeCount++; + break; + case RECEIVE: + recvCount++; + recvLen += event.getSecond(); + break; } - LOG.debug("recvCount={}", recvCount); - LOG.debug("recvLen={}", recvLen); - - assertEquals(2, connectCount); - // This test doesn't use actual PackedForward format so that it can simply test MultiSender itself. - // But w/o ack responses, Sender can't detect dropped requests. So some margin for expected result is allowed here. - long minExpectedRecvLen = ((long) (concurency * (sslEnabled ? 0.5 : 0.8)) * reqNum) * 10; - long maxExpectedRecvLen = ((long) concurency * reqNum) * 10; - assertThat(recvLen).isGreaterThanOrEqualTo(minExpectedRecvLen); - assertThat(recvLen).isLessThanOrEqualTo(maxExpectedRecvLen); - assertEquals(1, closeCount); + } } + LOG.debug("recvCount={}", recvCount); + LOG.debug("recvLen={}", recvLen); + + assertEquals(2, connectCount); + // This test doesn't use actual PackedForward format so that it can simply test MultiSender + // itself. + // But w/o ack responses, Sender can't detect dropped requests. So some margin for expected + // result is allowed here. + long minExpectedRecvLen = ((long) (concurency * (sslEnabled ? 0.5 : 0.8)) * reqNum) * 10; + long maxExpectedRecvLen = ((long) concurency * reqNum) * 10; + assertThat(recvLen).isGreaterThanOrEqualTo(minExpectedRecvLen); + assertThat(recvLen).isLessThanOrEqualTo(maxExpectedRecvLen); + assertEquals(1, closeCount); + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/RetryableSenderTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/RetryableSenderTest.java index f1307f21..33652c6c 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/RetryableSenderTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/RetryableSenderTest.java @@ -16,90 +16,81 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.fluentd.ingester.sender.retry.ExponentialBackOffRetryStrategy; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.fluentd.ingester.sender.retry.ExponentialBackOffRetryStrategy; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertThrows; - -class RetryableSenderTest -{ - @Test - void testSend() - throws IOException - { - ExponentialBackOffRetryStrategy.Config retryStrategyConfig = - new ExponentialBackOffRetryStrategy.Config(); - retryStrategyConfig.setMaxRetryCount(3); - - RetryableSender.Config senderConfig = new RetryableSender.Config(); - RetryableSender sender = new RetryableSender(senderConfig, - new FailurableSender(3), new ExponentialBackOffRetryStrategy(retryStrategyConfig)); - - FailurableSender baseSender = (FailurableSender) sender.getBaseSender(); - assertThat(baseSender.getRetry()).isEqualTo(0); - sender.send(ByteBuffer.allocate(64)); - assertThat(baseSender.getRetry()).isEqualTo(3); +class RetryableSenderTest { + @Test + void testSend() throws IOException { + ExponentialBackOffRetryStrategy.Config retryStrategyConfig = + new ExponentialBackOffRetryStrategy.Config(); + retryStrategyConfig.setMaxRetryCount(3); + + RetryableSender.Config senderConfig = new RetryableSender.Config(); + RetryableSender sender = + new RetryableSender( + senderConfig, + new FailurableSender(3), + new ExponentialBackOffRetryStrategy(retryStrategyConfig)); + + FailurableSender baseSender = (FailurableSender) sender.getBaseSender(); + assertThat(baseSender.getRetry()).isEqualTo(0); + sender.send(ByteBuffer.allocate(64)); + assertThat(baseSender.getRetry()).isEqualTo(3); + } + + @Test + void testSendRetryOver() throws IOException { + ExponentialBackOffRetryStrategy.Config retryStrategyConfig = + new ExponentialBackOffRetryStrategy.Config(); + retryStrategyConfig.setMaxRetryCount(2); + + RetryableSender.Config senderConfig = new RetryableSender.Config(); + RetryableSender sender = + new RetryableSender( + senderConfig, + new FailurableSender(3), + new ExponentialBackOffRetryStrategy(retryStrategyConfig)); + + FailurableSender baseSender = (FailurableSender) sender.getBaseSender(); + assertThat(baseSender.getRetry()).isEqualTo(0); + assertThrows( + RetryableSender.RetryOverException.class, () -> sender.send(ByteBuffer.allocate(64))); + } + + static class FailurableSender extends FluentdSender { + private final int maxFailures; + private int retry; + + FailurableSender(int maxFailures) { + super(new FluentdSender.Config()); + this.maxFailures = maxFailures; } - @Test - void testSendRetryOver() - throws IOException - { - ExponentialBackOffRetryStrategy.Config retryStrategyConfig = - new ExponentialBackOffRetryStrategy.Config(); - retryStrategyConfig.setMaxRetryCount(2); - - RetryableSender.Config senderConfig = new RetryableSender.Config(); - RetryableSender sender = new RetryableSender(senderConfig, - new FailurableSender(3), new ExponentialBackOffRetryStrategy(retryStrategyConfig)); - - FailurableSender baseSender = (FailurableSender) sender.getBaseSender(); - assertThat(baseSender.getRetry()).isEqualTo(0); - assertThrows(RetryableSender.RetryOverException.class, - () -> sender.send(ByteBuffer.allocate(64))); + @Override + public boolean isAvailable() { + return true; } - static class FailurableSender - extends FluentdSender - { - private final int maxFailures; - private int retry; - - FailurableSender(int maxFailures) - { - super(new FluentdSender.Config()); - this.maxFailures = maxFailures; - } - - @Override - public boolean isAvailable() - { - return true; - } - - @Override - protected void sendInternal(List buffers, String ackToken) - throws IOException - { - if (retry < maxFailures) { - retry++; - throw new IOException("Something is wrong..."); - } - } - - int getRetry() - { - return retry; - } + @Override + protected void sendInternal(List buffers, String ackToken) throws IOException { + if (retry < maxFailures) { + retry++; + throw new IOException("Something is wrong..."); + } + } - @Override - public void close() - { - } + int getRetry() { + return retry; } + + @Override + public void close() {} + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSenderTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSenderTest.java index c6191667..f9008358 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSenderTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSenderTest.java @@ -16,15 +16,12 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.fluentd.MockTCPServer; -import org.komamitsu.fluency.fluentd.MockTCPServerWithMetrics; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; -import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.SSLHeartbeater; -import org.komamitsu.fluency.util.Tuple; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; import java.io.IOException; import java.net.SocketException; @@ -36,241 +33,232 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.fluentd.MockTCPServer; +import org.komamitsu.fluency.fluentd.MockTCPServerWithMetrics; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; +import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.SSLHeartbeater; +import org.komamitsu.fluency.util.Tuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; - -class SSLSenderTest -{ - private static final Logger LOG = LoggerFactory.getLogger(SSLSenderTest.class); - - @Test - void testSend() - throws Exception - { - testSendBase(port -> { - SSLSender.Config config = new SSLSender.Config(); - config.setPort(port); - config.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - return new SSLSender(config); - }, - count -> assertThat(count).isEqualTo(1), - count -> assertThat(count).isEqualTo(1)); +class SSLSenderTest { + private static final Logger LOG = LoggerFactory.getLogger(SSLSenderTest.class); + + @Test + void testSend() throws Exception { + testSendBase( + port -> { + SSLSender.Config config = new SSLSender.Config(); + config.setPort(port); + config.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + return new SSLSender(config); + }, + count -> assertThat(count).isEqualTo(1), + count -> assertThat(count).isEqualTo(1)); + } + + @Test + void testSendWithHeartbeart() throws Exception { + testSendBase( + port -> { + SSLHeartbeater.Config hbConfig = new SSLHeartbeater.Config(); + hbConfig.setPort(port); + hbConfig.setIntervalMillis(400); + SSLSender.Config senderConfig = new SSLSender.Config(); + senderConfig.setPort(port); + senderConfig.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + return new SSLSender( + senderConfig, + new FailureDetector( + new PhiAccrualFailureDetectStrategy(), new SSLHeartbeater(hbConfig))); + }, + count -> assertThat(count).isGreaterThan(1), + count -> assertThat(count).isGreaterThan(1)); + } + + private void testSendBase( + SSLSenderCreator sslSenderCreator, + Consumer connectCountAssertion, + Consumer closeCountAssertion) + throws Exception { + MockTCPServerWithMetrics server = new MockTCPServerWithMetrics(true); + server.start(); + + int concurency = 20; + final int reqNum = 5000; + final CountDownLatch latch = new CountDownLatch(concurency); + SSLSender sender = sslSenderCreator.create(server.getLocalPort()); + + // To receive heartbeat at least once + TimeUnit.MILLISECONDS.sleep(500); + + final ExecutorService senderExecutorService = Executors.newCachedThreadPool(); + for (int i = 0; i < concurency; i++) { + senderExecutorService.execute( + () -> { + try { + byte[] bytes = "0123456789".getBytes(Charset.forName("UTF-8")); + + for (int j = 0; j < reqNum; j++) { + sender.send(ByteBuffer.wrap(bytes)); + } + latch.countDown(); + } catch (IOException e) { + e.printStackTrace(); + } + }); } - @Test - void testSendWithHeartbeart() - throws Exception - { - testSendBase(port -> { - SSLHeartbeater.Config hbConfig = new SSLHeartbeater.Config(); - hbConfig.setPort(port); - hbConfig.setIntervalMillis(400); - SSLSender.Config senderConfig = new SSLSender.Config(); - senderConfig.setPort(port); - senderConfig.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - return new SSLSender(senderConfig, - new FailureDetector( - new PhiAccrualFailureDetectStrategy(), - new SSLHeartbeater(hbConfig))); - }, - count -> assertThat(count).isGreaterThan(1), - count -> assertThat(count).isGreaterThan(1)); + if (!latch.await(30, TimeUnit.SECONDS)) { + fail("Sending all requests timed out"); } - - private void testSendBase( - SSLSenderCreator sslSenderCreator, - Consumer connectCountAssertion, - Consumer closeCountAssertion) - throws Exception - { - MockTCPServerWithMetrics server = new MockTCPServerWithMetrics(true); - server.start(); - - int concurency = 20; - final int reqNum = 5000; - final CountDownLatch latch = new CountDownLatch(concurency); - SSLSender sender = sslSenderCreator.create(server.getLocalPort()); - - // To receive heartbeat at least once - TimeUnit.MILLISECONDS.sleep(500); - - final ExecutorService senderExecutorService = Executors.newCachedThreadPool(); - for (int i = 0; i < concurency; i++) { - senderExecutorService.execute(() -> { - try { - byte[] bytes = "0123456789".getBytes(Charset.forName("UTF-8")); - - for (int j = 0; j < reqNum; j++) { - sender.send(ByteBuffer.wrap(bytes)); - } - latch.countDown(); - } - catch (IOException e) { - e.printStackTrace(); - } - }); - } - - if (!latch.await(30, TimeUnit.SECONDS)) { - fail("Sending all requests timed out"); - } - sender.close(); - - server.waitUntilEventsStop(); - server.stop(); - - int connectCount = 0; - int closeCount = 0; - long recvCount = 0; - long recvLen = 0; - for (Tuple event : server.getEvents()) { - switch (event.getFirst()) { - case CONNECT: - connectCount++; - break; - case CLOSE: - closeCount++; - break; - case RECEIVE: - recvCount++; - recvLen += event.getSecond(); - break; - } - } - LOG.debug("recvCount={}", recvCount); - - connectCountAssertion.accept(connectCount); - assertThat(recvLen).isEqualTo((long) concurency * reqNum * 10); - closeCountAssertion.accept(closeCount); + sender.close(); + + server.waitUntilEventsStop(); + server.stop(); + + int connectCount = 0; + int closeCount = 0; + long recvCount = 0; + long recvLen = 0; + for (Tuple event : server.getEvents()) { + switch (event.getFirst()) { + case CONNECT: + connectCount++; + break; + case CLOSE: + closeCount++; + break; + case RECEIVE: + recvCount++; + recvLen += event.getSecond(); + break; + } } - - @Test - void testConnectionTimeout() - throws IOException, InterruptedException - { - final CountDownLatch latch = new CountDownLatch(1); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.execute(() -> { + LOG.debug("recvCount={}", recvCount); + + connectCountAssertion.accept(connectCount); + assertThat(recvLen).isEqualTo((long) concurency * reqNum * 10); + closeCountAssertion.accept(closeCount); + } + + @Test + void testConnectionTimeout() throws IOException, InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.execute( + () -> { + SSLSender.Config senderConfig = new SSLSender.Config(); + senderConfig.setHost("192.0.2.0"); + senderConfig.setConnectionTimeoutMilli(1000); + senderConfig.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + SSLSender sender = new SSLSender(senderConfig); + try { + sender.send(ByteBuffer.wrap("hello, world".getBytes("UTF-8"))); + } catch (Throwable e) { + if (e instanceof SocketTimeoutException) { + latch.countDown(); + } else { + throw new RuntimeException(e); + } + } + }); + assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); + } + + @Test + void testReadTimeout() throws Exception { + final MockTCPServer server = new MockTCPServer(true); + server.start(); + + try { + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.execute( + () -> { SSLSender.Config senderConfig = new SSLSender.Config(); - senderConfig.setHost("192.0.2.0"); - senderConfig.setConnectionTimeoutMilli(1000); + senderConfig.setPort(server.getLocalPort()); + senderConfig.setReadTimeoutMilli(1000); senderConfig.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); SSLSender sender = new SSLSender(senderConfig); try { - sender.send(ByteBuffer.wrap("hello, world".getBytes("UTF-8"))); - } - catch (Throwable e) { - if (e instanceof SocketTimeoutException) { - latch.countDown(); - } - else { - throw new RuntimeException(e); - } + sender.sendWithAck( + Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), + "Waiting ack forever"); + } catch (Throwable e) { + if (e instanceof SocketTimeoutException) { + latch.countDown(); + } else { + throw new RuntimeException(e); + } } - }); - assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); + }); + assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); + } finally { + server.stop(); } + } - @Test - void testReadTimeout() - throws Exception - { - final MockTCPServer server = new MockTCPServer(true); - server.start(); - - try { - final CountDownLatch latch = new CountDownLatch(1); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.execute(() -> { - SSLSender.Config senderConfig = new SSLSender.Config(); - senderConfig.setPort(server.getLocalPort()); - senderConfig.setReadTimeoutMilli(1000); - senderConfig.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - SSLSender sender = new SSLSender(senderConfig); - try { - sender.sendWithAck(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), "Waiting ack forever"); - } - catch (Throwable e) { - if (e instanceof SocketTimeoutException) { - latch.countDown(); - } - else { - throw new RuntimeException(e); - } - } - }); - assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); - } - finally { - server.stop(); - } - } - - private Throwable extractRootCause(Throwable exception) - { - Throwable e = exception; - while (e.getCause() != null) { - e = e.getCause(); - } - return e; + private Throwable extractRootCause(Throwable exception) { + Throwable e = exception; + while (e.getCause() != null) { + e = e.getCause(); } + return e; + } + + @Test + void testDisconnBeforeRecv() throws Exception { + final MockTCPServer server = new MockTCPServer(true); + server.start(); + + try { + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.execute( + () -> { + SSLSender.Config senderConfig = new SSLSender.Config(); + senderConfig.setPort(server.getLocalPort()); + senderConfig.setReadTimeoutMilli(4000); + senderConfig.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); + SSLSender sender = new SSLSender(senderConfig); + try { + sender.sendWithAck( + Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), + "Waiting ack forever"); + } catch (Throwable e) { + Throwable rootCause = extractRootCause(e); + if (rootCause instanceof SocketException + && rootCause.getMessage().toLowerCase().contains("disconnected")) { + latch.countDown(); + } else { + throw new RuntimeException(e); + } + } + }); - @Test - void testDisconnBeforeRecv() - throws Exception - { - final MockTCPServer server = new MockTCPServer(true); - server.start(); - - try { - final CountDownLatch latch = new CountDownLatch(1); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.execute(() -> { - SSLSender.Config senderConfig = new SSLSender.Config(); - senderConfig.setPort(server.getLocalPort()); - senderConfig.setReadTimeoutMilli(4000); - senderConfig.setSslSocketFactory(SSL_CLIENT_SOCKET_FACTORY); - SSLSender sender = new SSLSender(senderConfig); - try { - sender.sendWithAck(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), "Waiting ack forever"); - } - catch (Throwable e) { - Throwable rootCause = extractRootCause(e); - if (rootCause instanceof SocketException && rootCause.getMessage().toLowerCase().contains("disconnected")) { - latch.countDown(); - } - else { - throw new RuntimeException(e); - } - } - }); - - TimeUnit.MILLISECONDS.sleep(1000); - server.stop(true); + TimeUnit.MILLISECONDS.sleep(1000); + server.stop(true); - assertTrue(latch.await(8000, TimeUnit.MILLISECONDS)); - } - finally { - server.stop(); - } + assertTrue(latch.await(8000, TimeUnit.MILLISECONDS)); + } finally { + server.stop(); } - - @Test - void testClose() - throws Exception - { - final MockTCPServer server = new MockTCPServer(true); - server.start(); - - try { - final AtomicLong duration = new AtomicLong(); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - Future future = executorService.submit(() -> { + } + + @Test + void testClose() throws Exception { + final MockTCPServer server = new MockTCPServer(true); + server.start(); + + try { + final AtomicLong duration = new AtomicLong(); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + Future future = + executorService.submit( + () -> { SSLSender.Config senderConfig = new SSLSender.Config(); senderConfig.setPort(server.getLocalPort()); senderConfig.setWaitBeforeCloseMilli(1500); @@ -278,57 +266,52 @@ void testClose() SSLSender sender = new SSLSender(senderConfig); long start; try { - sender.send(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes("UTF-8")))); - start = System.currentTimeMillis(); - sender.close(); - duration.set(System.currentTimeMillis() - start); - } - catch (Exception e) { - LOG.error("Unexpected exception", e); + sender.send(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes("UTF-8")))); + start = System.currentTimeMillis(); + sender.close(); + duration.set(System.currentTimeMillis() - start); + } catch (Exception e) { + LOG.error("Unexpected exception", e); } return null; - }); - future.get(3000, TimeUnit.MILLISECONDS); - assertTrue(duration.get() > 1000 && duration.get() < 2000); - } - finally { - server.stop(); - } + }); + future.get(3000, TimeUnit.MILLISECONDS); + assertTrue(duration.get() > 1000 && duration.get() < 2000); + } finally { + server.stop(); } + } + + @Test + void testConfig() { + SSLSender.Config config = new SSLSender.Config(); + assertEquals(1000, config.getWaitBeforeCloseMilli()); + // TODO: Add others later + } - @Test - void testConfig() + @Test + void validateConfig() { { - SSLSender.Config config = new SSLSender.Config(); - assertEquals(1000, config.getWaitBeforeCloseMilli()); - // TODO: Add others later + SSLSender.Config config = new SSLSender.Config(); + config.setConnectionTimeoutMilli(9); + assertThrows(IllegalArgumentException.class, () -> new SSLSender(config)); } - @Test - void validateConfig() { - { - SSLSender.Config config = new SSLSender.Config(); - config.setConnectionTimeoutMilli(9); - assertThrows(IllegalArgumentException.class, () -> new SSLSender(config)); - } - - { - SSLSender.Config config = new SSLSender.Config(); - config.setReadTimeoutMilli(9); - assertThrows(IllegalArgumentException.class, () -> new SSLSender(config)); - } - - { - SSLSender.Config config = new SSLSender.Config(); - config.setWaitBeforeCloseMilli(-1); - assertThrows(IllegalArgumentException.class, () -> new SSLSender(config)); - } + SSLSender.Config config = new SSLSender.Config(); + config.setReadTimeoutMilli(9); + assertThrows(IllegalArgumentException.class, () -> new SSLSender(config)); } - interface SSLSenderCreator { - SSLSender create(int port); + SSLSender.Config config = new SSLSender.Config(); + config.setWaitBeforeCloseMilli(-1); + assertThrows(IllegalArgumentException.class, () -> new SSLSender(config)); } -} \ No newline at end of file + } + + interface SSLSenderCreator { + SSLSender create(int port); + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilderTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilderTest.java index c3fd8650..cce42d84 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilderTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilderTest.java @@ -16,13 +16,8 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.fluentd.SSLTestSocketFactories; - -import javax.net.ssl.SSLServerSocket; -import javax.net.ssl.SSLSocket; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; import java.io.IOException; import java.io.OutputStream; @@ -40,65 +35,69 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import javax.net.ssl.SSLServerSocket; +import javax.net.ssl.SSLSocket; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.fluentd.SSLTestSocketFactories; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.komamitsu.fluency.fluentd.SSLTestSocketFactories.SSL_CLIENT_SOCKET_FACTORY; - -class SSLSocketBuilderTest -{ - private SSLServerSocket serverSocket; +class SSLSocketBuilderTest { + private SSLServerSocket serverSocket; - @BeforeEach - void setUp() - throws IOException, CertificateException, NoSuchAlgorithmException, UnrecoverableKeyException, KeyStoreException, KeyManagementException - { - serverSocket = SSLTestSocketFactories.createServerSocket(); - } + @BeforeEach + void setUp() + throws IOException, + CertificateException, + NoSuchAlgorithmException, + UnrecoverableKeyException, + KeyStoreException, + KeyManagementException { + serverSocket = SSLTestSocketFactories.createServerSocket(); + } - @AfterEach - void tearDown() - throws IOException - { - if (serverSocket != null) { - serverSocket.close(); - } + @AfterEach + void tearDown() throws IOException { + if (serverSocket != null) { + serverSocket.close(); } + } - @Test - void testWithServer() - throws IOException, InterruptedException, ExecutionException, TimeoutException - { - final AtomicInteger readLen = new AtomicInteger(); - final byte[] buf = new byte[256]; + @Test + void testWithServer() + throws IOException, InterruptedException, ExecutionException, TimeoutException { + final AtomicInteger readLen = new AtomicInteger(); + final byte[] buf = new byte[256]; - ExecutorService executorService = Executors.newSingleThreadExecutor(); - Future future = executorService.submit(new Callable() - { - @Override - public Void call() - throws Exception - { + ExecutorService executorService = Executors.newSingleThreadExecutor(); + Future future = + executorService.submit( + new Callable() { + @Override + public Void call() throws Exception { Socket clientSocket = serverSocket.accept(); readLen.set(clientSocket.getInputStream().read(buf)); return null; - } - }); + } + }); - SSLSocket sslSocket = new SSLSocketBuilder("localhost", serverSocket.getLocalPort(), 5000, 5000, SSL_CLIENT_SOCKET_FACTORY).build(); + SSLSocket sslSocket = + new SSLSocketBuilder( + "localhost", serverSocket.getLocalPort(), 5000, 5000, SSL_CLIENT_SOCKET_FACTORY) + .build(); - try { - OutputStream outputStream = sslSocket.getOutputStream(); - outputStream.write("hello".getBytes("ASCII")); - outputStream.flush(); - } - finally { - sslSocket.close(); - } + try { + OutputStream outputStream = sslSocket.getOutputStream(); + outputStream.write("hello".getBytes("ASCII")); + outputStream.flush(); + } finally { + sslSocket.close(); + } - future.get(10, TimeUnit.SECONDS); - executorService.shutdown(); - executorService.awaitTermination(10, TimeUnit.SECONDS); + future.get(10, TimeUnit.SECONDS); + executorService.shutdown(); + executorService.awaitTermination(10, TimeUnit.SECONDS); - assertEquals(new String(buf, 0, readLen.get(), "ASCII"), "hello"); - } + assertEquals(new String(buf, 0, readLen.get(), "ASCII"), "hello"); + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/TCPSenderTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/TCPSenderTest.java index cd6bfa92..6826c554 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/TCPSenderTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/TCPSenderTest.java @@ -16,15 +16,10 @@ package org.komamitsu.fluency.fluentd.ingester.sender; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.fluentd.MockTCPServer; -import org.komamitsu.fluency.fluentd.MockTCPServerWithMetrics; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; -import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; -import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.TCPHeartbeater; -import org.komamitsu.fluency.util.Tuple; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.net.SocketException; @@ -40,291 +35,279 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.fluentd.MockTCPServer; +import org.komamitsu.fluency.fluentd.MockTCPServerWithMetrics; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.FailureDetector; +import org.komamitsu.fluency.fluentd.ingester.sender.failuredetect.PhiAccrualFailureDetectStrategy; +import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.TCPHeartbeater; +import org.komamitsu.fluency.util.Tuple; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class TCPSenderTest -{ - private static final Logger LOG = LoggerFactory.getLogger(TCPSenderTest.class); - - @Test - void testSend() - throws Exception - { - testSendBase(port -> { - TCPSender.Config senderConfig = new TCPSender.Config(); - senderConfig.setPort(port); - return new TCPSender(senderConfig); - }, - count -> assertThat(count).isEqualTo(1), - count -> assertThat(count).isEqualTo(1)); +class TCPSenderTest { + private static final Logger LOG = LoggerFactory.getLogger(TCPSenderTest.class); + + @Test + void testSend() throws Exception { + testSendBase( + port -> { + TCPSender.Config senderConfig = new TCPSender.Config(); + senderConfig.setPort(port); + return new TCPSender(senderConfig); + }, + count -> assertThat(count).isEqualTo(1), + count -> assertThat(count).isEqualTo(1)); + } + + @Test + void testSendWithHeartbeart() throws Exception { + testSendBase( + port -> { + TCPHeartbeater.Config hbConfig = new TCPHeartbeater.Config(); + hbConfig.setPort(port); + hbConfig.setIntervalMillis(400); + + TCPSender.Config senderConfig = new TCPSender.Config(); + senderConfig.setPort(port); + + return new TCPSender( + senderConfig, + new FailureDetector( + new PhiAccrualFailureDetectStrategy(), new TCPHeartbeater(hbConfig))); + }, + count -> assertThat(count).isGreaterThan(1), + count -> assertThat(count).isGreaterThan(1)); + } + + private void testSendBase( + TCPSenderCreater senderCreater, + Consumer connectCountAssertion, + Consumer closeCountAssertion) + throws Exception { + MockTCPServerWithMetrics server = new MockTCPServerWithMetrics(false); + server.start(); + + int concurency = 20; + final int reqNum = 5000; + final CountDownLatch latch = new CountDownLatch(concurency); + TCPSender sender = senderCreater.create(server.getLocalPort()); + + // To receive heartbeat at least once + TimeUnit.MILLISECONDS.sleep(500); + + final ExecutorService senderExecutorService = Executors.newCachedThreadPool(); + for (int i = 0; i < concurency; i++) { + senderExecutorService.execute( + () -> { + try { + byte[] bytes = "0123456789".getBytes(Charset.forName("UTF-8")); + + for (int j = 0; j < reqNum; j++) { + sender.send(ByteBuffer.wrap(bytes)); + } + latch.countDown(); + } catch (IOException e) { + e.printStackTrace(); + } + }); } - @Test - void testSendWithHeartbeart() - throws Exception - { - testSendBase(port -> { - TCPHeartbeater.Config hbConfig = new TCPHeartbeater.Config(); - hbConfig.setPort(port); - hbConfig.setIntervalMillis(400); - - TCPSender.Config senderConfig = new TCPSender.Config(); - senderConfig.setPort(port); - - return new TCPSender(senderConfig, - new FailureDetector( - new PhiAccrualFailureDetectStrategy(), - new TCPHeartbeater(hbConfig))); - }, - count -> assertThat(count).isGreaterThan(1), - count -> assertThat(count).isGreaterThan(1)); + assertTrue(latch.await(4, TimeUnit.SECONDS)); + sender.close(); + + server.waitUntilEventsStop(); + server.stop(); + + int connectCount = 0; + int closeCount = 0; + long recvCount = 0; + long recvLen = 0; + for (Tuple event : server.getEvents()) { + switch (event.getFirst()) { + case CONNECT: + connectCount++; + break; + case CLOSE: + closeCount++; + break; + case RECEIVE: + recvCount++; + recvLen += event.getSecond(); + break; + } } - - private void testSendBase( - TCPSenderCreater senderCreater, - Consumer connectCountAssertion, - Consumer closeCountAssertion) - throws Exception - { - MockTCPServerWithMetrics server = new MockTCPServerWithMetrics(false); - server.start(); - - int concurency = 20; - final int reqNum = 5000; - final CountDownLatch latch = new CountDownLatch(concurency); - TCPSender sender = senderCreater.create(server.getLocalPort()); - - // To receive heartbeat at least once - TimeUnit.MILLISECONDS.sleep(500); - - final ExecutorService senderExecutorService = Executors.newCachedThreadPool(); - for (int i = 0; i < concurency; i++) { - senderExecutorService.execute(() -> { - try { - byte[] bytes = "0123456789".getBytes(Charset.forName("UTF-8")); - - for (int j = 0; j < reqNum; j++) { - sender.send(ByteBuffer.wrap(bytes)); - } - latch.countDown(); - } - catch (IOException e) { - e.printStackTrace(); - } - }); - } - - assertTrue(latch.await(4, TimeUnit.SECONDS)); - sender.close(); - - server.waitUntilEventsStop(); - server.stop(); - - int connectCount = 0; - int closeCount = 0; - long recvCount = 0; - long recvLen = 0; - for (Tuple event : server.getEvents()) { - switch (event.getFirst()) { - case CONNECT: - connectCount++; - break; - case CLOSE: - closeCount++; - break; - case RECEIVE: - recvCount++; - recvLen += event.getSecond(); - break; + LOG.debug("recvCount={}", recvCount); + + connectCountAssertion.accept(connectCount); + assertThat(recvLen).isEqualTo((long) concurency * reqNum * 10); + closeCountAssertion.accept(closeCount); + } + + @Test + void testConnectionTimeout() throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.execute( + () -> { + TCPSender.Config senderConfig = new TCPSender.Config(); + senderConfig.setHost("192.0.2.0"); + senderConfig.setConnectionTimeoutMilli(1000); + TCPSender sender = new TCPSender(senderConfig); + try { + sender.send(ByteBuffer.wrap("hello, world".getBytes("UTF-8"))); + } catch (Throwable e) { + if (e instanceof SocketTimeoutException) { + latch.countDown(); + } else { + throw new RuntimeException(e); } - } - LOG.debug("recvCount={}", recvCount); - - connectCountAssertion.accept(connectCount); - assertThat(recvLen).isEqualTo((long) concurency * reqNum * 10); - closeCountAssertion.accept(closeCount); - } - - @Test - void testConnectionTimeout() - throws InterruptedException - { - final CountDownLatch latch = new CountDownLatch(1); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.execute(() -> { + } + }); + assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); + } + + @Test + void testReadTimeout() throws Exception { + final MockTCPServer server = new MockTCPServer(false); + server.start(); + + try { + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.execute( + () -> { TCPSender.Config senderConfig = new TCPSender.Config(); - senderConfig.setHost("192.0.2.0"); - senderConfig.setConnectionTimeoutMilli(1000); + senderConfig.setPort(server.getLocalPort()); + senderConfig.setReadTimeoutMilli(1000); TCPSender sender = new TCPSender(senderConfig); try { - sender.send(ByteBuffer.wrap("hello, world".getBytes("UTF-8"))); + sender.sendWithAck( + Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), + "Waiting ack forever"); + } catch (Throwable e) { + if (e instanceof SocketTimeoutException) { + latch.countDown(); + } else { + throw new RuntimeException(e); + } } - catch (Throwable e) { - if (e instanceof SocketTimeoutException) { - latch.countDown(); - } - else { - throw new RuntimeException(e); - } - } - }); - assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); + }); + assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); + } finally { + server.stop(); } + } - @Test - void testReadTimeout() - throws Exception - { - final MockTCPServer server = new MockTCPServer(false); - server.start(); - - try { - final CountDownLatch latch = new CountDownLatch(1); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.execute(() -> { - TCPSender.Config senderConfig = new TCPSender.Config(); - senderConfig.setPort(server.getLocalPort()); - senderConfig.setReadTimeoutMilli(1000); - TCPSender sender = new TCPSender(senderConfig); - try { - sender.sendWithAck(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), "Waiting ack forever"); - } - catch (Throwable e) { - if (e instanceof SocketTimeoutException) { - latch.countDown(); - } - else { - throw new RuntimeException(e); - } - } - }); - assertTrue(latch.await(2000, TimeUnit.MILLISECONDS)); - } - finally { - server.stop(); - } + private Throwable extractRootCause(Throwable exception) { + Throwable e = exception; + while (e.getCause() != null) { + e = e.getCause(); } + return e; + } + + @Test + void testDisconnBeforeRecv() throws Exception { + final MockTCPServer server = new MockTCPServer(false); + server.start(); + + try { + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + executorService.execute( + () -> { + TCPSender.Config senderConfig = new TCPSender.Config(); + senderConfig.setPort(server.getLocalPort()); + senderConfig.setReadTimeoutMilli(4000); + TCPSender sender = new TCPSender(senderConfig); + try { + sender.sendWithAck( + Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), + "Waiting ack forever"); + } catch (Throwable e) { + Throwable rootCause = extractRootCause(e); + if (rootCause instanceof SocketException + && rootCause.getMessage().toLowerCase().contains("disconnected")) { + latch.countDown(); + } else { + throw new RuntimeException(e); + } + } + }); - private Throwable extractRootCause(Throwable exception) - { - Throwable e = exception; - while (e.getCause() != null) { - e = e.getCause(); - } - return e; - } - - @Test - void testDisconnBeforeRecv() - throws Exception - { - final MockTCPServer server = new MockTCPServer(false); - server.start(); - - try { - final CountDownLatch latch = new CountDownLatch(1); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.execute(() -> { - TCPSender.Config senderConfig = new TCPSender.Config(); - senderConfig.setPort(server.getLocalPort()); - senderConfig.setReadTimeoutMilli(4000); - TCPSender sender = new TCPSender(senderConfig); - try { - sender.sendWithAck(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes(StandardCharsets.UTF_8))), "Waiting ack forever"); - } - catch (Throwable e) { - Throwable rootCause = extractRootCause(e); - if (rootCause instanceof SocketException && rootCause.getMessage().toLowerCase().contains("disconnected")) { - latch.countDown(); - } - else { - throw new RuntimeException(e); - } - } - }); - - TimeUnit.MILLISECONDS.sleep(1000); - server.stop(true); + TimeUnit.MILLISECONDS.sleep(1000); + server.stop(true); - assertTrue(latch.await(8000, TimeUnit.MILLISECONDS)); - } - finally { - server.stop(); - } + assertTrue(latch.await(8000, TimeUnit.MILLISECONDS)); + } finally { + server.stop(); } - - @Test - void testClose() - throws Exception - { - final MockTCPServer server = new MockTCPServer(false); - server.start(); - - try { - final AtomicLong duration = new AtomicLong(); - ExecutorService executorService = Executors.newSingleThreadExecutor(); - Future future = executorService.submit(() -> { + } + + @Test + void testClose() throws Exception { + final MockTCPServer server = new MockTCPServer(false); + server.start(); + + try { + final AtomicLong duration = new AtomicLong(); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + Future future = + executorService.submit( + () -> { TCPSender.Config senderConfig = new TCPSender.Config(); senderConfig.setPort(server.getLocalPort()); senderConfig.setWaitBeforeCloseMilli(1500); TCPSender sender = new TCPSender(senderConfig); long start; try { - sender.send(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes("UTF-8")))); - start = System.currentTimeMillis(); - sender.close(); - duration.set(System.currentTimeMillis() - start); - } - catch (Exception e) { - LOG.error("Unexpected exception", e); + sender.send(Arrays.asList(ByteBuffer.wrap("hello, world".getBytes("UTF-8")))); + start = System.currentTimeMillis(); + sender.close(); + duration.set(System.currentTimeMillis() - start); + } catch (Exception e) { + LOG.error("Unexpected exception", e); } return null; - }); - future.get(3000, TimeUnit.MILLISECONDS); - assertTrue(duration.get() > 1000 && duration.get() < 2000); - } - finally { - server.stop(); - } + }); + future.get(3000, TimeUnit.MILLISECONDS); + assertTrue(duration.get() > 1000 && duration.get() < 2000); + } finally { + server.stop(); } + } - @Test - void testConfig() + @Test + void testConfig() { + TCPSender.Config config = new TCPSender.Config(); + assertEquals(1000, config.getWaitBeforeCloseMilli()); + // TODO: Add others later + } + + @Test + void validateConfig() { { - TCPSender.Config config = new TCPSender.Config(); - assertEquals(1000, config.getWaitBeforeCloseMilli()); - // TODO: Add others later + TCPSender.Config config = new TCPSender.Config(); + config.setConnectionTimeoutMilli(9); + assertThrows(IllegalArgumentException.class, () -> new TCPSender(config)); } - @Test - void validateConfig() { - { - TCPSender.Config config = new TCPSender.Config(); - config.setConnectionTimeoutMilli(9); - assertThrows(IllegalArgumentException.class, () -> new TCPSender(config)); - } - - { - TCPSender.Config config = new TCPSender.Config(); - config.setReadTimeoutMilli(9); - assertThrows(IllegalArgumentException.class, () -> new TCPSender(config)); - } - - { - TCPSender.Config config = new TCPSender.Config(); - config.setWaitBeforeCloseMilli(-1); - assertThrows(IllegalArgumentException.class, () -> new TCPSender(config)); - } + TCPSender.Config config = new TCPSender.Config(); + config.setReadTimeoutMilli(9); + assertThrows(IllegalArgumentException.class, () -> new TCPSender(config)); } - interface TCPSenderCreater { - TCPSender create(int port); + TCPSender.Config config = new TCPSender.Config(); + config.setWaitBeforeCloseMilli(-1); + assertThrows(IllegalArgumentException.class, () -> new TCPSender(config)); } -} \ No newline at end of file + } + + interface TCPSenderCreater { + TCPSender create(int port); + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetectorTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetectorTest.java index 57344fa8..da276c2a 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetectorTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/FailureDetectorTest.java @@ -16,11 +16,10 @@ package org.komamitsu.fluency.fluentd.ingester.sender.failuredetect; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.Heartbeater; -import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.TCPHeartbeater; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; import java.io.IOException; import java.nio.channels.ServerSocketChannel; @@ -28,77 +27,74 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.Heartbeater; +import org.komamitsu.fluency.fluentd.ingester.sender.heartbeat.TCPHeartbeater; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; - -class FailureDetectorTest -{ - private static final Logger LOG = LoggerFactory.getLogger(FailureDetectorTest.class); +class FailureDetectorTest { + private static final Logger LOG = LoggerFactory.getLogger(FailureDetectorTest.class); - @Test - void testIsAvailable() - throws IOException, InterruptedException - { - final ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); - serverSocketChannel.socket().bind(null); - int localPort = serverSocketChannel.socket().getLocalPort(); + @Test + void testIsAvailable() throws IOException, InterruptedException { + final ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); + serverSocketChannel.socket().bind(null); + int localPort = serverSocketChannel.socket().getLocalPort(); - final ExecutorService executorService = Executors.newSingleThreadExecutor(); - Runnable serverTask = () -> { - while (!executorService.isShutdown()) { - try (SocketChannel accept = serverSocketChannel.accept()) { - LOG.debug("Accepted: {}", accept); - } - catch (IOException e) { - LOG.warn("Stab TCP server got an error", e); - } - } - try { - serverSocketChannel.close(); - } - catch (IOException e) { - LOG.warn("Failed to close serverSocketChannel", e); + final ExecutorService executorService = Executors.newSingleThreadExecutor(); + Runnable serverTask = + () -> { + while (!executorService.isShutdown()) { + try (SocketChannel accept = serverSocketChannel.accept()) { + LOG.debug("Accepted: {}", accept); + } catch (IOException e) { + LOG.warn("Stab TCP server got an error", e); } + } + try { + serverSocketChannel.close(); + } catch (IOException e) { + LOG.warn("Failed to close serverSocketChannel", e); + } }; - executorService.execute(serverTask); + executorService.execute(serverTask); - TCPHeartbeater.Config heartbeaterConfig = new TCPHeartbeater.Config(); - heartbeaterConfig.setPort(localPort); + TCPHeartbeater.Config heartbeaterConfig = new TCPHeartbeater.Config(); + heartbeaterConfig.setPort(localPort); - PhiAccrualFailureDetectStrategy.Config failureDetectorConfig = new PhiAccrualFailureDetectStrategy.Config(); - try (FailureDetector failureDetector = new FailureDetector( - new PhiAccrualFailureDetectStrategy(failureDetectorConfig), - new TCPHeartbeater(heartbeaterConfig))) { + PhiAccrualFailureDetectStrategy.Config failureDetectorConfig = + new PhiAccrualFailureDetectStrategy.Config(); + try (FailureDetector failureDetector = + new FailureDetector( + new PhiAccrualFailureDetectStrategy(failureDetectorConfig), + new TCPHeartbeater(heartbeaterConfig))) { - assertTrue(failureDetector.isAvailable()); - TimeUnit.SECONDS.sleep(4); - assertTrue(failureDetector.isAvailable()); + assertTrue(failureDetector.isAvailable()); + TimeUnit.SECONDS.sleep(4); + assertTrue(failureDetector.isAvailable()); - executorService.shutdownNow(); - for (int i = 0; i < 20; i++) { - if (!failureDetector.isAvailable()) { - break; - } - TimeUnit.MILLISECONDS.sleep(500); - } - assertFalse(failureDetector.isAvailable()); + executorService.shutdownNow(); + for (int i = 0; i < 20; i++) { + if (!failureDetector.isAvailable()) { + break; } + TimeUnit.MILLISECONDS.sleep(500); + } + assertFalse(failureDetector.isAvailable()); } + } - @Test - void validateConfig() - { - FailureDetector.Config config = new FailureDetector.Config(); - config.setFailureIntervalMillis(-1); + @Test + void validateConfig() { + FailureDetector.Config config = new FailureDetector.Config(); + config.setFailureIntervalMillis(-1); - assertThrows(IllegalArgumentException.class, () -> new FailureDetector( - mock(FailureDetectStrategy.class), - mock(Heartbeater.class), - config - )); - } -} \ No newline at end of file + assertThrows( + IllegalArgumentException.class, + () -> + new FailureDetector( + mock(FailureDetectStrategy.class), mock(Heartbeater.class), config)); + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/PhiAccrualFailureDetectStrategyTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/PhiAccrualFailureDetectStrategyTest.java index 0b9d3fde..0b4f06dc 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/PhiAccrualFailureDetectStrategyTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/failuredetect/PhiAccrualFailureDetectStrategyTest.java @@ -16,25 +16,25 @@ package org.komamitsu.fluency.fluentd.ingester.sender.failuredetect; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.*; -class PhiAccrualFailureDetectStrategyTest -{ - @Test - void validateConfig() +import org.junit.jupiter.api.Test; + +class PhiAccrualFailureDetectStrategyTest { + @Test + void validateConfig() { { - { - PhiAccrualFailureDetectStrategy.Config config = new PhiAccrualFailureDetectStrategy.Config(); - config.setPhiThreshold(0); - assertThrows(IllegalArgumentException.class, () -> new PhiAccrualFailureDetectStrategy(config)); - } + PhiAccrualFailureDetectStrategy.Config config = new PhiAccrualFailureDetectStrategy.Config(); + config.setPhiThreshold(0); + assertThrows( + IllegalArgumentException.class, () -> new PhiAccrualFailureDetectStrategy(config)); + } - { - PhiAccrualFailureDetectStrategy.Config config = new PhiAccrualFailureDetectStrategy.Config(); - config.setArrivalWindowSize(0); - assertThrows(IllegalArgumentException.class, () -> new PhiAccrualFailureDetectStrategy(config)); - } + { + PhiAccrualFailureDetectStrategy.Config config = new PhiAccrualFailureDetectStrategy.Config(); + config.setArrivalWindowSize(0); + assertThrows( + IllegalArgumentException.class, () -> new PhiAccrualFailureDetectStrategy(config)); } -} \ No newline at end of file + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/HeartbeaterTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/HeartbeaterTest.java index 99546d66..922c8721 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/HeartbeaterTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/HeartbeaterTest.java @@ -16,91 +16,77 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.util.Tuple3; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -public class HeartbeaterTest -{ - private TestableHeartbeater.Config config; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.util.Tuple3; - @BeforeEach - public void setup() - { - config = new TestableHeartbeater.Config(); - config.setHost("dummy-hostname"); - config.setPort(123456); - config.setIntervalMillis(300); - } +public class HeartbeaterTest { + private TestableHeartbeater.Config config; - @Test - public void testHeartbeater() - throws InterruptedException - { - try (TestableHeartbeater heartbeater = new TestableHeartbeater(config)) { - heartbeater.start(); - TimeUnit.SECONDS.sleep(1); - heartbeater.close(); - TimeUnit.MILLISECONDS.sleep(500); - assertTrue(1 < heartbeater.pongedRecords.size() && heartbeater.pongedRecords.size() < 4); - Tuple3 firstPong = heartbeater.pongedRecords.get(0); - Tuple3 secondPong = heartbeater.pongedRecords.get(1); - long diff = secondPong.getFirst() - firstPong.getFirst(); - assertTrue(100 < diff && diff < 1000); - for (Tuple3 pongedRecord : heartbeater.pongedRecords) { - assertEquals("dummy-hostname", pongedRecord.getSecond()); - assertEquals((Integer) 123456, pongedRecord.getThird()); - } - } - } + @BeforeEach + public void setup() { + config = new TestableHeartbeater.Config(); + config.setHost("dummy-hostname"); + config.setPort(123456); + config.setIntervalMillis(300); + } - @Test - void validateConfig() - { - TestableHeartbeater.Config invalidConfig = new TestableHeartbeater.Config(); - invalidConfig.setIntervalMillis(99); - assertThrows(IllegalArgumentException.class, () -> new TestableHeartbeater(invalidConfig)); + @Test + public void testHeartbeater() throws InterruptedException { + try (TestableHeartbeater heartbeater = new TestableHeartbeater(config)) { + heartbeater.start(); + TimeUnit.SECONDS.sleep(1); + heartbeater.close(); + TimeUnit.MILLISECONDS.sleep(500); + assertTrue(1 < heartbeater.pongedRecords.size() && heartbeater.pongedRecords.size() < 4); + Tuple3 firstPong = heartbeater.pongedRecords.get(0); + Tuple3 secondPong = heartbeater.pongedRecords.get(1); + long diff = secondPong.getFirst() - firstPong.getFirst(); + assertTrue(100 < diff && diff < 1000); + for (Tuple3 pongedRecord : heartbeater.pongedRecords) { + assertEquals("dummy-hostname", pongedRecord.getSecond()); + assertEquals((Integer) 123456, pongedRecord.getThird()); + } } + } - private static class TestableHeartbeater - extends InetSocketHeartbeater - { - private final Config config; - private List> pongedRecords = new ArrayList<>(); + @Test + void validateConfig() { + TestableHeartbeater.Config invalidConfig = new TestableHeartbeater.Config(); + invalidConfig.setIntervalMillis(99); + assertThrows(IllegalArgumentException.class, () -> new TestableHeartbeater(invalidConfig)); + } - public TestableHeartbeater() - { - this(new Config()); - } + private static class TestableHeartbeater extends InetSocketHeartbeater { + private final Config config; + private List> pongedRecords = new ArrayList<>(); - public TestableHeartbeater(Config config) - { - super(config); - this.config = config; - } + public TestableHeartbeater() { + this(new Config()); + } - public List> getPongedRecords() - { - return pongedRecords; - } + public TestableHeartbeater(Config config) { + super(config); + this.config = config; + } - @Override - protected void invoke() - { - pongedRecords.add(new Tuple3<>(System.currentTimeMillis(), config.getHost(), config.getPort())); - } + public List> getPongedRecords() { + return pongedRecords; + } - private static class Config - extends InetSocketHeartbeater.Config - { - } + @Override + protected void invoke() { + pongedRecords.add( + new Tuple3<>(System.currentTimeMillis(), config.getHost(), config.getPort())); } -} \ No newline at end of file + + private static class Config extends InetSocketHeartbeater.Config {} + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeaterTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeaterTest.java index ea324a15..2d51dfea 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeaterTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeaterTest.java @@ -16,12 +16,9 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.fluentd.SSLTestSocketFactories; - -import javax.net.ssl.SSLServerSocket; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.security.KeyManagementException; @@ -33,136 +30,128 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import javax.net.ssl.SSLServerSocket; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.fluentd.SSLTestSocketFactories; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class SSLHeartbeaterTest -{ - private SSLServerSocket sslServerSocket; - private SSLHeartbeater heartbeater; +public class SSLHeartbeaterTest { + private SSLServerSocket sslServerSocket; + private SSLHeartbeater heartbeater; + + @BeforeEach + void setUp() + throws CertificateException, + UnrecoverableKeyException, + NoSuchAlgorithmException, + IOException, + KeyManagementException, + KeyStoreException { + sslServerSocket = SSLTestSocketFactories.createServerSocket(); + + SSLHeartbeater.Config config = new SSLHeartbeater.Config(); + config.setPort(sslServerSocket.getLocalPort()); + config.setIntervalMillis(500); + heartbeater = new SSLHeartbeater(config); + } + + @AfterEach + void tearDown() throws IOException { + if (sslServerSocket != null && !sslServerSocket.isClosed()) { + sslServerSocket.close(); + } + } + + @Test + void testHeartbeaterUp() throws IOException, InterruptedException { + final CountDownLatch latch = new CountDownLatch(2); + Executors.newSingleThreadExecutor() + .execute( + () -> { + try { + sslServerSocket.accept(); + latch.countDown(); + } catch (IOException e) { + e.printStackTrace(); + } + }); - @BeforeEach - void setUp() - throws CertificateException, UnrecoverableKeyException, NoSuchAlgorithmException, IOException, KeyManagementException, KeyStoreException - { - sslServerSocket = SSLTestSocketFactories.createServerSocket(); + final AtomicInteger pongCounter = new AtomicInteger(); + final AtomicInteger failureCounter = new AtomicInteger(); + try { + heartbeater.setCallback( + new Heartbeater.Callback() { + @Override + public void onHeartbeat() { + pongCounter.incrementAndGet(); + latch.countDown(); + } - SSLHeartbeater.Config config = new SSLHeartbeater.Config(); - config.setPort(sslServerSocket.getLocalPort()); - config.setIntervalMillis(500); - heartbeater = new SSLHeartbeater(config); + @Override + public void onFailure(Throwable cause) { + failureCounter.incrementAndGet(); + } + }); + heartbeater.start(); + assertTrue(latch.await(10, TimeUnit.SECONDS)); + assertTrue(0 < pongCounter.get() && pongCounter.get() < 3); + assertEquals(0, failureCounter.get()); + } finally { + if (heartbeater != null) { + heartbeater.close(); + } } + } + + @Test + public void testHeartbeaterDown() throws IOException, InterruptedException { + sslServerSocket.close(); + + final AtomicInteger pongCounter = new AtomicInteger(); + final AtomicInteger failureCounter = new AtomicInteger(); + try { + heartbeater.setCallback( + new Heartbeater.Callback() { + @Override + public void onHeartbeat() { + pongCounter.incrementAndGet(); + } - @AfterEach - void tearDown() - throws IOException - { - if (sslServerSocket != null && !sslServerSocket.isClosed()) { - sslServerSocket.close(); - } + @Override + public void onFailure(Throwable cause) { + failureCounter.incrementAndGet(); + } + }); + heartbeater.start(); + TimeUnit.SECONDS.sleep(1); + assertEquals(0, pongCounter.get()); + assertTrue(failureCounter.get() > 0); + } finally { + if (heartbeater != null) { + heartbeater.close(); + } } + } - @Test - void testHeartbeaterUp() - throws IOException, InterruptedException + @Test + void validateConfig() { { - final CountDownLatch latch = new CountDownLatch(2); - Executors.newSingleThreadExecutor().execute(() -> { - try { - sslServerSocket.accept(); - latch.countDown(); - } - catch (IOException e) { - e.printStackTrace(); - } - }); - - final AtomicInteger pongCounter = new AtomicInteger(); - final AtomicInteger failureCounter = new AtomicInteger(); - try { - heartbeater.setCallback(new Heartbeater.Callback() - { - @Override - public void onHeartbeat() - { - pongCounter.incrementAndGet(); - latch.countDown(); - } - - @Override - public void onFailure(Throwable cause) - { - failureCounter.incrementAndGet(); - } - }); - heartbeater.start(); - assertTrue(latch.await(10, TimeUnit.SECONDS)); - assertTrue(0 < pongCounter.get() && pongCounter.get() < 3); - assertEquals(0, failureCounter.get()); - } - finally { - if (heartbeater != null) { - heartbeater.close(); - } - } + SSLHeartbeater.Config config = new SSLHeartbeater.Config(); + config.setIntervalMillis(99); + assertThrows(IllegalArgumentException.class, () -> new SSLHeartbeater(config)); } - @Test - public void testHeartbeaterDown() - throws IOException, InterruptedException { - sslServerSocket.close(); - - final AtomicInteger pongCounter = new AtomicInteger(); - final AtomicInteger failureCounter = new AtomicInteger(); - try { - heartbeater.setCallback(new Heartbeater.Callback() - { - @Override - public void onHeartbeat() - { - pongCounter.incrementAndGet(); - } - - @Override - public void onFailure(Throwable cause) - { - failureCounter.incrementAndGet(); - } - }); - heartbeater.start(); - TimeUnit.SECONDS.sleep(1); - assertEquals(0, pongCounter.get()); - assertTrue(failureCounter.get() > 0); - } - finally { - if (heartbeater != null) { - heartbeater.close(); - } - } + SSLHeartbeater.Config config = new SSLHeartbeater.Config(); + config.setConnectionTimeoutMilli(9); + assertThrows(IllegalArgumentException.class, () -> new SSLHeartbeater(config)); } - @Test - void validateConfig() { - { - SSLHeartbeater.Config config = new SSLHeartbeater.Config(); - config.setIntervalMillis(99); - assertThrows(IllegalArgumentException.class, () -> new SSLHeartbeater(config)); - } - - { - SSLHeartbeater.Config config = new SSLHeartbeater.Config(); - config.setConnectionTimeoutMilli(9); - assertThrows(IllegalArgumentException.class, () -> new SSLHeartbeater(config)); - } - - { - SSLHeartbeater.Config config = new SSLHeartbeater.Config(); - config.setReadTimeoutMilli(9); - assertThrows(IllegalArgumentException.class, () -> new SSLHeartbeater(config)); - } + SSLHeartbeater.Config config = new SSLHeartbeater.Config(); + config.setReadTimeoutMilli(9); + assertThrows(IllegalArgumentException.class, () -> new SSLHeartbeater(config)); } -} \ No newline at end of file + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/TCPHeartbeaterTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/TCPHeartbeaterTest.java index a121860f..111f8abc 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/TCPHeartbeaterTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/TCPHeartbeaterTest.java @@ -16,7 +16,8 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.nio.channels.ServerSocketChannel; @@ -24,90 +25,80 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class TCPHeartbeaterTest -{ - @Test - void testTCPHeartbeaterUp() - throws IOException, InterruptedException - { - final CountDownLatch latch = new CountDownLatch(2); - final ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); - serverSocketChannel.socket().bind(null); - Executors.newSingleThreadExecutor().execute(() -> { - try { +class TCPHeartbeaterTest { + @Test + void testTCPHeartbeaterUp() throws IOException, InterruptedException { + final CountDownLatch latch = new CountDownLatch(2); + final ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); + serverSocketChannel.socket().bind(null); + Executors.newSingleThreadExecutor() + .execute( + () -> { + try { serverSocketChannel.accept(); latch.countDown(); - } - catch (IOException e) { + } catch (IOException e) { e.printStackTrace(); - } - }); + } + }); - TCPHeartbeater.Config config = new TCPHeartbeater.Config(); - config.setPort(serverSocketChannel.socket().getLocalPort()); - config.setIntervalMillis(500); - try (TCPHeartbeater heartbeater = new TCPHeartbeater(config)) { - final AtomicInteger pongCounter = new AtomicInteger(); - final AtomicInteger failureCounter = new AtomicInteger(); - heartbeater.setCallback(new Heartbeater.Callback() - { - @Override - public void onHeartbeat() - { - pongCounter.incrementAndGet(); - latch.countDown(); - } + TCPHeartbeater.Config config = new TCPHeartbeater.Config(); + config.setPort(serverSocketChannel.socket().getLocalPort()); + config.setIntervalMillis(500); + try (TCPHeartbeater heartbeater = new TCPHeartbeater(config)) { + final AtomicInteger pongCounter = new AtomicInteger(); + final AtomicInteger failureCounter = new AtomicInteger(); + heartbeater.setCallback( + new Heartbeater.Callback() { + @Override + public void onHeartbeat() { + pongCounter.incrementAndGet(); + latch.countDown(); + } - @Override - public void onFailure(Throwable cause) - { - failureCounter.incrementAndGet(); - } - }); - heartbeater.start(); - assertTrue(latch.await(5, TimeUnit.SECONDS)); - assertTrue(0 < pongCounter.get() && pongCounter.get() < 3); - assertEquals(0, failureCounter.get()); - } + @Override + public void onFailure(Throwable cause) { + failureCounter.incrementAndGet(); + } + }); + heartbeater.start(); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(0 < pongCounter.get() && pongCounter.get() < 3); + assertEquals(0, failureCounter.get()); } + } - @Test - void testTCPHeartbeaterDown() - throws IOException, InterruptedException - { - final ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); - serverSocketChannel.socket().bind(null); - int localPort = serverSocketChannel.socket().getLocalPort(); - serverSocketChannel.close(); + @Test + void testTCPHeartbeaterDown() throws IOException, InterruptedException { + final ServerSocketChannel serverSocketChannel = ServerSocketChannel.open(); + serverSocketChannel.socket().bind(null); + int localPort = serverSocketChannel.socket().getLocalPort(); + serverSocketChannel.close(); - TCPHeartbeater.Config config = new TCPHeartbeater.Config(); - config.setPort(localPort); - config.setIntervalMillis(500); - try (TCPHeartbeater heartbeater = new TCPHeartbeater(config)) { - final AtomicInteger pongCounter = new AtomicInteger(); - final AtomicInteger failureCounter = new AtomicInteger(); - heartbeater.setCallback(new Heartbeater.Callback() - { - @Override - public void onHeartbeat() - { - pongCounter.incrementAndGet(); - } + TCPHeartbeater.Config config = new TCPHeartbeater.Config(); + config.setPort(localPort); + config.setIntervalMillis(500); + try (TCPHeartbeater heartbeater = new TCPHeartbeater(config)) { + final AtomicInteger pongCounter = new AtomicInteger(); + final AtomicInteger failureCounter = new AtomicInteger(); + heartbeater.setCallback( + new Heartbeater.Callback() { + @Override + public void onHeartbeat() { + pongCounter.incrementAndGet(); + } - @Override - public void onFailure(Throwable cause) - { - failureCounter.incrementAndGet(); - } - }); - heartbeater.start(); - TimeUnit.SECONDS.sleep(1); - assertEquals(0, pongCounter.get()); - assertTrue(failureCounter.get() > 0); - } + @Override + public void onFailure(Throwable cause) { + failureCounter.incrementAndGet(); + } + }); + heartbeater.start(); + TimeUnit.SECONDS.sleep(1); + assertEquals(0, pongCounter.get()); + assertTrue(failureCounter.get() > 0); } + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UDPHeartbeaterTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UDPHeartbeaterTest.java index 1e299bdf..5d92e322 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UDPHeartbeaterTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/UDPHeartbeaterTest.java @@ -16,7 +16,8 @@ package org.komamitsu.fluency.fluentd.ingester.sender.heartbeat; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.net.SocketAddress; @@ -26,89 +27,78 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class UDPHeartbeaterTest -{ - @Test - void testUDPHeartbeaterUp() - throws IOException, InterruptedException - { - final CountDownLatch latch = new CountDownLatch(2); - final DatagramChannel datagramChannel = DatagramChannel.open(); - datagramChannel.socket().bind(null); - Executors.newSingleThreadExecutor().execute(() -> { - try { +class UDPHeartbeaterTest { + @Test + void testUDPHeartbeaterUp() throws IOException, InterruptedException { + final CountDownLatch latch = new CountDownLatch(2); + final DatagramChannel datagramChannel = DatagramChannel.open(); + datagramChannel.socket().bind(null); + Executors.newSingleThreadExecutor() + .execute( + () -> { + try { ByteBuffer byteBuffer = ByteBuffer.allocate(8); SocketAddress socketAddress = datagramChannel.receive(byteBuffer); datagramChannel.send(ByteBuffer.allocate(0), socketAddress); latch.countDown(); - } - catch (IOException e) { + } catch (IOException e) { e.printStackTrace(); - } - }); + } + }); - UDPHeartbeater.Config config = new UDPHeartbeater.Config(); - config.setPort(datagramChannel.socket().getLocalPort()); - config.setIntervalMillis(500); - try (UDPHeartbeater heartbeater = new UDPHeartbeater(config)) { - final AtomicInteger pongCounter = new AtomicInteger(); - final AtomicInteger failureCounter = new AtomicInteger(); - heartbeater.setCallback(new Heartbeater.Callback() - { - @Override - public void onHeartbeat() - { - pongCounter.incrementAndGet(); - latch.countDown(); - } + UDPHeartbeater.Config config = new UDPHeartbeater.Config(); + config.setPort(datagramChannel.socket().getLocalPort()); + config.setIntervalMillis(500); + try (UDPHeartbeater heartbeater = new UDPHeartbeater(config)) { + final AtomicInteger pongCounter = new AtomicInteger(); + final AtomicInteger failureCounter = new AtomicInteger(); + heartbeater.setCallback( + new Heartbeater.Callback() { + @Override + public void onHeartbeat() { + pongCounter.incrementAndGet(); + latch.countDown(); + } - @Override - public void onFailure(Throwable cause) - { - failureCounter.incrementAndGet(); - } - }); - heartbeater.start(); - assertTrue(latch.await(5, TimeUnit.SECONDS)); - assertTrue(0 < pongCounter.get() && pongCounter.get() < 3); - assertEquals(0, failureCounter.get()); - } + @Override + public void onFailure(Throwable cause) { + failureCounter.incrementAndGet(); + } + }); + heartbeater.start(); + assertTrue(latch.await(5, TimeUnit.SECONDS)); + assertTrue(0 < pongCounter.get() && pongCounter.get() < 3); + assertEquals(0, failureCounter.get()); } + } - @Test - void testUDPHeartbeaterDown() - throws IOException, InterruptedException - { - final DatagramChannel datagramChannel = DatagramChannel.open(); - datagramChannel.socket().bind(null); - int localPort = datagramChannel.socket().getLocalPort(); - datagramChannel.close(); + @Test + void testUDPHeartbeaterDown() throws IOException, InterruptedException { + final DatagramChannel datagramChannel = DatagramChannel.open(); + datagramChannel.socket().bind(null); + int localPort = datagramChannel.socket().getLocalPort(); + datagramChannel.close(); - UDPHeartbeater.Config config = new UDPHeartbeater.Config(); - config.setPort(localPort); - config.setIntervalMillis(500); - try (UDPHeartbeater heartbeater = new UDPHeartbeater(config)) { - final AtomicInteger pongCounter = new AtomicInteger(); - heartbeater.setCallback(new Heartbeater.Callback() - { - @Override - public void onHeartbeat() - { - pongCounter.incrementAndGet(); - } + UDPHeartbeater.Config config = new UDPHeartbeater.Config(); + config.setPort(localPort); + config.setIntervalMillis(500); + try (UDPHeartbeater heartbeater = new UDPHeartbeater(config)) { + final AtomicInteger pongCounter = new AtomicInteger(); + heartbeater.setCallback( + new Heartbeater.Callback() { + @Override + public void onHeartbeat() { + pongCounter.incrementAndGet(); + } - @Override - public void onFailure(Throwable cause) - { - } - }); - heartbeater.start(); - TimeUnit.SECONDS.sleep(1); - assertEquals(0, pongCounter.get()); - } + @Override + public void onFailure(Throwable cause) {} + }); + heartbeater.start(); + TimeUnit.SECONDS.sleep(1); + assertEquals(0, pongCounter.get()); } + } } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ConstantRetryStrategyTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ConstantRetryStrategyTest.java index 99d1c38c..51d7975e 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ConstantRetryStrategyTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ConstantRetryStrategyTest.java @@ -16,38 +16,34 @@ package org.komamitsu.fluency.fluentd.ingester.sender.retry; - -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -class ConstantRetryStrategyTest -{ - @Test - void testGetNextIntervalMillis() - { - ConstantRetryStrategy.Config config = new ConstantRetryStrategy.Config(); - config.setRetryIntervalMillis(600); - config.setMaxRetryCount(6); - RetryStrategy strategy = new ConstantRetryStrategy(config); +import org.junit.jupiter.api.Test; + +class ConstantRetryStrategyTest { + @Test + void testGetNextIntervalMillis() { + ConstantRetryStrategy.Config config = new ConstantRetryStrategy.Config(); + config.setRetryIntervalMillis(600); + config.setMaxRetryCount(6); + RetryStrategy strategy = new ConstantRetryStrategy(config); - assertEquals(600, strategy.getNextIntervalMillis(0)); - assertEquals(600, strategy.getNextIntervalMillis(1)); - assertEquals(600, strategy.getNextIntervalMillis(2)); - assertEquals(600, strategy.getNextIntervalMillis(6)); - assertFalse(strategy.isRetriedOver(6)); - assertEquals(600, strategy.getNextIntervalMillis(7)); - assertTrue(strategy.isRetriedOver(7)); - } + assertEquals(600, strategy.getNextIntervalMillis(0)); + assertEquals(600, strategy.getNextIntervalMillis(1)); + assertEquals(600, strategy.getNextIntervalMillis(2)); + assertEquals(600, strategy.getNextIntervalMillis(6)); + assertFalse(strategy.isRetriedOver(6)); + assertEquals(600, strategy.getNextIntervalMillis(7)); + assertTrue(strategy.isRetriedOver(7)); + } - @Test - void validateConfig() - { - ConstantRetryStrategy.Config config = new ConstantRetryStrategy.Config(); - config.setRetryIntervalMillis(9); - assertThrows(IllegalArgumentException.class, () -> new ConstantRetryStrategy(config)); - } -} \ No newline at end of file + @Test + void validateConfig() { + ConstantRetryStrategy.Config config = new ConstantRetryStrategy.Config(); + config.setRetryIntervalMillis(9); + assertThrows(IllegalArgumentException.class, () -> new ConstantRetryStrategy(config)); + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ExponentialBackOffRetryStrategyTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ExponentialBackOffRetryStrategyTest.java index 9aa1fdc0..9be162f8 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ExponentialBackOffRetryStrategyTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/retry/ExponentialBackOffRetryStrategyTest.java @@ -16,54 +16,54 @@ package org.komamitsu.fluency.fluentd.ingester.sender.retry; -import org.junit.jupiter.api.Test; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -class ExponentialBackOffRetryStrategyTest -{ - @Test - void testGetNextIntervalMillis() - { - ExponentialBackOffRetryStrategy.Config config = new ExponentialBackOffRetryStrategy.Config(); - config.setBaseIntervalMillis(400); - config.setMaxIntervalMillis(30000); - config.setMaxRetryCount(7); - RetryStrategy strategy = new ExponentialBackOffRetryStrategy(config); +import org.junit.jupiter.api.Test; - assertEquals(400, strategy.getNextIntervalMillis(0)); - assertEquals(800, strategy.getNextIntervalMillis(1)); - assertEquals(1600, strategy.getNextIntervalMillis(2)); - assertEquals(3200, strategy.getNextIntervalMillis(3)); - assertEquals(25600, strategy.getNextIntervalMillis(6)); - assertEquals(30000, strategy.getNextIntervalMillis(7)); - assertFalse(strategy.isRetriedOver(7)); - assertEquals(30000, strategy.getNextIntervalMillis(8)); - assertTrue(strategy.isRetriedOver(8)); - } +class ExponentialBackOffRetryStrategyTest { + @Test + void testGetNextIntervalMillis() { + ExponentialBackOffRetryStrategy.Config config = new ExponentialBackOffRetryStrategy.Config(); + config.setBaseIntervalMillis(400); + config.setMaxIntervalMillis(30000); + config.setMaxRetryCount(7); + RetryStrategy strategy = new ExponentialBackOffRetryStrategy(config); - @Test - void validateConfig() + assertEquals(400, strategy.getNextIntervalMillis(0)); + assertEquals(800, strategy.getNextIntervalMillis(1)); + assertEquals(1600, strategy.getNextIntervalMillis(2)); + assertEquals(3200, strategy.getNextIntervalMillis(3)); + assertEquals(25600, strategy.getNextIntervalMillis(6)); + assertEquals(30000, strategy.getNextIntervalMillis(7)); + assertFalse(strategy.isRetriedOver(7)); + assertEquals(30000, strategy.getNextIntervalMillis(8)); + assertTrue(strategy.isRetriedOver(8)); + } + + @Test + void validateConfig() { { - { - ExponentialBackOffRetryStrategy.Config config = new ExponentialBackOffRetryStrategy.Config(); - config.setMaxRetryCount(-1); - assertThrows(IllegalArgumentException.class, () -> new ExponentialBackOffRetryStrategy(config)); - } + ExponentialBackOffRetryStrategy.Config config = new ExponentialBackOffRetryStrategy.Config(); + config.setMaxRetryCount(-1); + assertThrows( + IllegalArgumentException.class, () -> new ExponentialBackOffRetryStrategy(config)); + } - { - ExponentialBackOffRetryStrategy.Config config = new ExponentialBackOffRetryStrategy.Config(); - config.setBaseIntervalMillis(9); - assertThrows(IllegalArgumentException.class, () -> new ExponentialBackOffRetryStrategy(config)); - } + { + ExponentialBackOffRetryStrategy.Config config = new ExponentialBackOffRetryStrategy.Config(); + config.setBaseIntervalMillis(9); + assertThrows( + IllegalArgumentException.class, () -> new ExponentialBackOffRetryStrategy(config)); + } - { - ExponentialBackOffRetryStrategy.Config config = new ExponentialBackOffRetryStrategy.Config(); - config.setMaxIntervalMillis(9); - assertThrows(IllegalArgumentException.class, () -> new ExponentialBackOffRetryStrategy(config)); - } + { + ExponentialBackOffRetryStrategy.Config config = new ExponentialBackOffRetryStrategy.Config(); + config.setMaxIntervalMillis(9); + assertThrows( + IllegalArgumentException.class, () -> new ExponentialBackOffRetryStrategy(config)); } -} \ No newline at end of file + } +} diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/integration/WithRealFluentd.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/integration/WithRealFluentd.java index e832bb54..b957563a 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/integration/WithRealFluentd.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/integration/WithRealFluentd.java @@ -16,12 +16,10 @@ package org.komamitsu.fluency.integration; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.Fluency; -import org.komamitsu.fluency.fluentd.FluencyBuilderForFluentd; - import java.io.IOException; import java.net.InetSocketAddress; import java.util.ArrayList; @@ -36,203 +34,191 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.Fluency; +import org.komamitsu.fluency.fluentd.FluencyBuilderForFluentd; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -class WithRealFluentd -{ - private final ObjectMapper objectMapper = new ObjectMapper(); +class WithRealFluentd { + private final ObjectMapper objectMapper = new ObjectMapper(); - WithRealFluentd.Config getConfig() - throws IOException - { - String conf = System.getenv("WITH_FLUENTD"); - if (conf == null) { - return null; - } - - return objectMapper.readValue(conf, WithRealFluentd.Config.class); + WithRealFluentd.Config getConfig() throws IOException { + String conf = System.getenv("WITH_FLUENTD"); + if (conf == null) { + return null; } - @Test - void testWithRealFluentd() - throws Exception - { - WithRealFluentd.Config config = getConfig(); - assumeTrue(config != null); - - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(config.sslEnabled); - - try (Fluency fluency = builder.build(config.host, config.port)) { - Map data = new HashMap<>(); - data.put("name", "komamitsu"); - data.put("age", 42); - data.put("comment", "hello, world"); - ExecutorService executorService = Executors.newCachedThreadPool(); - List> futures = new ArrayList<>(); - for (int i = 0; i < config.concurrency; i++) { - futures.add(executorService.submit(new EmitTask(fluency, config.tag, data, config.requests))); - } - for (Future future : futures) { - future.get(config.waitSeconds, TimeUnit.SECONDS); - } - } + return objectMapper.readValue(conf, WithRealFluentd.Config.class); + } + + @Test + void testWithRealFluentd() throws Exception { + WithRealFluentd.Config config = getConfig(); + assumeTrue(config != null); + + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(config.sslEnabled); + + try (Fluency fluency = builder.build(config.host, config.port)) { + Map data = new HashMap<>(); + data.put("name", "komamitsu"); + data.put("age", 42); + data.put("comment", "hello, world"); + ExecutorService executorService = Executors.newCachedThreadPool(); + List> futures = new ArrayList<>(); + for (int i = 0; i < config.concurrency; i++) { + futures.add( + executorService.submit(new EmitTask(fluency, config.tag, data, config.requests))); + } + for (Future future : futures) { + future.get(config.waitSeconds, TimeUnit.SECONDS); + } } - - @Test - void testWithRealMultipleFluentd() - throws IOException, InterruptedException, TimeoutException, ExecutionException - { - WithRealFluentd.Config config = getConfig(); - assumeTrue(config != null); - assumeTrue(config.anotherPort != null); - - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(config.sslEnabled); - builder.setAckResponseMode(true); - - try (Fluency fluency = builder.build( - Arrays.asList( - new InetSocketAddress(config.host, config.port), - new InetSocketAddress(config.host, config.anotherPort)))) { - - Map data = new HashMap<>(); - data.put("name", "komamitsu"); - data.put("age", 42); - data.put("comment", "hello, world"); - ExecutorService executorService = Executors.newCachedThreadPool(); - List> futures = new ArrayList<>(); - for (int i = 0; i < config.concurrency; i++) { - futures.add(executorService.submit(new EmitTask(fluency, config.tag, data, config.requests))); - } - for (Future future : futures) { - future.get(config.waitSeconds, TimeUnit.SECONDS); - } - } + } + + @Test + void testWithRealMultipleFluentd() + throws IOException, InterruptedException, TimeoutException, ExecutionException { + WithRealFluentd.Config config = getConfig(); + assumeTrue(config != null); + assumeTrue(config.anotherPort != null); + + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(config.sslEnabled); + builder.setAckResponseMode(true); + + try (Fluency fluency = + builder.build( + Arrays.asList( + new InetSocketAddress(config.host, config.port), + new InetSocketAddress(config.host, config.anotherPort)))) { + + Map data = new HashMap<>(); + data.put("name", "komamitsu"); + data.put("age", 42); + data.put("comment", "hello, world"); + ExecutorService executorService = Executors.newCachedThreadPool(); + List> futures = new ArrayList<>(); + for (int i = 0; i < config.concurrency; i++) { + futures.add( + executorService.submit(new EmitTask(fluency, config.tag, data, config.requests))); + } + for (Future future : futures) { + future.get(config.waitSeconds, TimeUnit.SECONDS); + } } - - @Test - void testWithRealFluentdWithFileBackup() - throws ExecutionException, TimeoutException, IOException, InterruptedException - { - WithRealFluentd.Config config = getConfig(); - assumeTrue(config != null); - - FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); - builder.setSslEnabled(config.sslEnabled); - // Fluency might use a lot of buffer for loaded backup files. - // So it'd better increase max buffer size - builder.setMaxBufferSize(512 * 1024 * 1024L); - builder.setFileBackupDir(System.getProperty("java.io.tmpdir")); - - try (Fluency fluency = new FluencyBuilderForFluentd().build(config.host, config.port)) { - Map data = new HashMap<>(); - data.put("name", "komamitsu"); - data.put("age", 42); - data.put("comment", "hello, world"); - ExecutorService executorService = Executors.newCachedThreadPool(); - List> futures = new ArrayList<>(); - for (int i = 0; i < config.concurrency; i++) { - futures.add(executorService.submit(new EmitTask(fluency, config.tag, data, config.requests))); - } - for (Future future : futures) { - future.get(config.waitSeconds, TimeUnit.SECONDS); - } - } + } + + @Test + void testWithRealFluentdWithFileBackup() + throws ExecutionException, TimeoutException, IOException, InterruptedException { + WithRealFluentd.Config config = getConfig(); + assumeTrue(config != null); + + FluencyBuilderForFluentd builder = new FluencyBuilderForFluentd(); + builder.setSslEnabled(config.sslEnabled); + // Fluency might use a lot of buffer for loaded backup files. + // So it'd better increase max buffer size + builder.setMaxBufferSize(512 * 1024 * 1024L); + builder.setFileBackupDir(System.getProperty("java.io.tmpdir")); + + try (Fluency fluency = new FluencyBuilderForFluentd().build(config.host, config.port)) { + Map data = new HashMap<>(); + data.put("name", "komamitsu"); + data.put("age", 42); + data.put("comment", "hello, world"); + ExecutorService executorService = Executors.newCachedThreadPool(); + List> futures = new ArrayList<>(); + for (int i = 0; i < config.concurrency; i++) { + futures.add( + executorService.submit(new EmitTask(fluency, config.tag, data, config.requests))); + } + for (Future future : futures) { + future.get(config.waitSeconds, TimeUnit.SECONDS); + } } - - public static class EmitTask - implements Callable - { - private final Fluency fluency; - private final String tag; - private final Map data; - private final int count; - - private EmitTask(Fluency fluency, String tag, Map data, int count) - { - this.fluency = fluency; - this.tag = tag; - this.data = data; - this.count = count; - } - - @Override - public Void call() - { - for (int i = 0; i < count; i++) { - try { - fluency.emit(tag, data); - } - catch (IOException e) { - e.printStackTrace(); - try { - TimeUnit.MILLISECONDS.sleep(500); - } - catch (InterruptedException e1) { - e1.printStackTrace(); - } - } - } - return null; - } + } + + public static class EmitTask implements Callable { + private final Fluency fluency; + private final String tag; + private final Map data; + private final int count; + + private EmitTask(Fluency fluency, String tag, Map data, int count) { + this.fluency = fluency; + this.tag = tag; + this.data = data; + this.count = count; } - public static class Config - { - @JsonProperty("host") - public final String host; - @JsonProperty("port") - public final Integer port; - @JsonProperty("another_host") - public final String anotherHost; - @JsonProperty("another_port") - public final Integer anotherPort; - @JsonProperty("tag") - public final String tag; - @JsonProperty("requests") - public final int requests; - @JsonProperty("concurrency") - public final int concurrency; - @JsonProperty("wait_seconds") - public final int waitSeconds; - @JsonProperty("ssl_enabled") - public final boolean sslEnabled; - - public Config( - @JsonProperty("host") - String host, - @JsonProperty("port") - Integer port, - @JsonProperty("another_host") - String anotherHost, - @JsonProperty("another_port") - Integer anotherPort, - @JsonProperty("tag") - String tag, - @JsonProperty("requests") - Integer requests, - @JsonProperty("concurrency") - Integer concurrency, - @JsonProperty("wait_seconds") - Integer waitSeconds, - @JsonProperty("ssl_enabled") - Boolean sslEnabled - ) - { - this.host = host == null ? "127.0.0.1" : host; - this.port = port == null ? Integer.valueOf(24224) : port; - - this.anotherHost = anotherHost == null ? "127.0.0.1" : anotherHost; - // Nullable - this.anotherPort = anotherPort; - - this.tag = tag == null ? "foodb.bartbl" : tag; - this.requests = requests == null ? 1000000 : requests; - this.concurrency = concurrency == null ? 4 : concurrency; - this.waitSeconds = waitSeconds == null ? 60 : waitSeconds; - this.sslEnabled = sslEnabled == null ? false : sslEnabled; + @Override + public Void call() { + for (int i = 0; i < count; i++) { + try { + fluency.emit(tag, data); + } catch (IOException e) { + e.printStackTrace(); + try { + TimeUnit.MILLISECONDS.sleep(500); + } catch (InterruptedException e1) { + e1.printStackTrace(); + } } + } + return null; + } + } + + public static class Config { + @JsonProperty("host") + public final String host; + + @JsonProperty("port") + public final Integer port; + + @JsonProperty("another_host") + public final String anotherHost; + + @JsonProperty("another_port") + public final Integer anotherPort; + + @JsonProperty("tag") + public final String tag; + + @JsonProperty("requests") + public final int requests; + + @JsonProperty("concurrency") + public final int concurrency; + + @JsonProperty("wait_seconds") + public final int waitSeconds; + + @JsonProperty("ssl_enabled") + public final boolean sslEnabled; + + public Config( + @JsonProperty("host") String host, + @JsonProperty("port") Integer port, + @JsonProperty("another_host") String anotherHost, + @JsonProperty("another_port") Integer anotherPort, + @JsonProperty("tag") String tag, + @JsonProperty("requests") Integer requests, + @JsonProperty("concurrency") Integer concurrency, + @JsonProperty("wait_seconds") Integer waitSeconds, + @JsonProperty("ssl_enabled") Boolean sslEnabled) { + this.host = host == null ? "127.0.0.1" : host; + this.port = port == null ? Integer.valueOf(24224) : port; + + this.anotherHost = anotherHost == null ? "127.0.0.1" : anotherHost; + // Nullable + this.anotherPort = anotherPort; + + this.tag = tag == null ? "foodb.bartbl" : tag; + this.requests = requests == null ? 1000000 : requests; + this.concurrency = concurrency == null ? 4 : concurrency; + this.waitSeconds = waitSeconds == null ? 60 : waitSeconds; + this.sslEnabled = sslEnabled == null ? false : sslEnabled; } + } } diff --git a/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/FluencyBuilderForTreasureData.java b/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/FluencyBuilderForTreasureData.java index dc8bd5bf..50e43f8e 100644 --- a/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/FluencyBuilderForTreasureData.java +++ b/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/FluencyBuilderForTreasureData.java @@ -18,152 +18,136 @@ import org.komamitsu.fluency.Fluency; import org.komamitsu.fluency.ingester.Ingester; -import org.komamitsu.fluency.recordformat.RecordFormatter; +import org.komamitsu.fluency.recordformat.MessagePackRecordFormatter; import org.komamitsu.fluency.treasuredata.ingester.TreasureDataIngester; import org.komamitsu.fluency.treasuredata.ingester.sender.TreasureDataSender; -import org.komamitsu.fluency.recordformat.MessagePackRecordFormatter; -public class FluencyBuilderForTreasureData - extends org.komamitsu.fluency.FluencyBuilder -{ - private Integer senderRetryMax; - private Integer senderRetryIntervalMillis; - private Integer senderMaxRetryIntervalMillis; - private Float senderRetryFactor; - private Integer senderWorkBufSize; - - public FluencyBuilderForTreasureData() - { - setBufferChunkRetentionTimeMillis(30 * 1000); - setBufferChunkInitialSize(4 * 1024 * 1024); - setBufferChunkRetentionSize(64 * 1024 * 1024); - } +public class FluencyBuilderForTreasureData extends org.komamitsu.fluency.FluencyBuilder { + private Integer senderRetryMax; + private Integer senderRetryIntervalMillis; + private Integer senderMaxRetryIntervalMillis; + private Float senderRetryFactor; + private Integer senderWorkBufSize; - public Integer getSenderRetryMax() - { - return senderRetryMax; - } + public FluencyBuilderForTreasureData() { + setBufferChunkRetentionTimeMillis(30 * 1000); + setBufferChunkInitialSize(4 * 1024 * 1024); + setBufferChunkRetentionSize(64 * 1024 * 1024); + } - public void setSenderRetryMax(Integer senderRetryMax) - { - this.senderRetryMax = senderRetryMax; - } + public Integer getSenderRetryMax() { + return senderRetryMax; + } - public Integer getSenderRetryIntervalMillis() - { - return senderRetryIntervalMillis; - } + public void setSenderRetryMax(Integer senderRetryMax) { + this.senderRetryMax = senderRetryMax; + } - public void setSenderRetryIntervalMillis(Integer senderRetryIntervalMillis) - { - this.senderRetryIntervalMillis = senderRetryIntervalMillis; - } + public Integer getSenderRetryIntervalMillis() { + return senderRetryIntervalMillis; + } - public Integer getSenderMaxRetryIntervalMillis() - { - return senderMaxRetryIntervalMillis; - } - - public void setSenderMaxRetryIntervalMillis(Integer senderMaxRetryIntervalMillis) - { - this.senderMaxRetryIntervalMillis = senderMaxRetryIntervalMillis; - } + public void setSenderRetryIntervalMillis(Integer senderRetryIntervalMillis) { + this.senderRetryIntervalMillis = senderRetryIntervalMillis; + } - public Float getSenderRetryFactor() - { - return senderRetryFactor; - } - - public void setSenderRetryFactor(Float senderRetryFactor) - { - this.senderRetryFactor = senderRetryFactor; - } + public Integer getSenderMaxRetryIntervalMillis() { + return senderMaxRetryIntervalMillis; + } - public Integer getSenderWorkBufSize() - { - return senderWorkBufSize; - } + public void setSenderMaxRetryIntervalMillis(Integer senderMaxRetryIntervalMillis) { + this.senderMaxRetryIntervalMillis = senderMaxRetryIntervalMillis; + } - public void setSenderWorkBufSize(Integer senderWorkBufSize) - { - this.senderWorkBufSize = senderWorkBufSize; - } + public Float getSenderRetryFactor() { + return senderRetryFactor; + } - public Fluency build(String apikey, String endpoint) - { - return buildFromIngester( - buildRecordFormatter(), - buildIngester(createSenderConfig(endpoint, apikey))); - } + public void setSenderRetryFactor(Float senderRetryFactor) { + this.senderRetryFactor = senderRetryFactor; + } - public Fluency build(String apikey) - { - return buildFromIngester( - buildRecordFormatter(), - buildIngester(createSenderConfig(null, apikey))); - } + public Integer getSenderWorkBufSize() { + return senderWorkBufSize; + } - private TreasureDataSender.Config createSenderConfig(String endpoint, String apikey) - { - if (apikey == null) { - throw new IllegalArgumentException("`apikey` should be set"); - } + public void setSenderWorkBufSize(Integer senderWorkBufSize) { + this.senderWorkBufSize = senderWorkBufSize; + } - TreasureDataSender.Config senderConfig = new TreasureDataSender.Config(); - senderConfig.setApikey(apikey); + public Fluency build(String apikey, String endpoint) { + return buildFromIngester( + buildRecordFormatter(), buildIngester(createSenderConfig(endpoint, apikey))); + } - if (endpoint != null) { - senderConfig.setEndpoint(endpoint); - } + public Fluency build(String apikey) { + return buildFromIngester( + buildRecordFormatter(), buildIngester(createSenderConfig(null, apikey))); + } - if (getSenderRetryMax() != null) { - senderConfig.setRetryMax(getSenderRetryMax()); - } + private TreasureDataSender.Config createSenderConfig(String endpoint, String apikey) { + if (apikey == null) { + throw new IllegalArgumentException("`apikey` should be set"); + } - if (getSenderRetryIntervalMillis() != null) { - senderConfig.setRetryIntervalMs(getSenderRetryIntervalMillis()); - } + TreasureDataSender.Config senderConfig = new TreasureDataSender.Config(); + senderConfig.setApikey(apikey); - if (getSenderMaxRetryIntervalMillis() != null) { - senderConfig.setMaxRetryIntervalMs(getSenderMaxRetryIntervalMillis()); - } + if (endpoint != null) { + senderConfig.setEndpoint(endpoint); + } - if (getSenderRetryFactor() != null) { - senderConfig.setRetryFactor(getSenderRetryFactor()); - } + if (getSenderRetryMax() != null) { + senderConfig.setRetryMax(getSenderRetryMax()); + } - if (getErrorHandler() != null) { - senderConfig.setErrorHandler(getErrorHandler()); - } + if (getSenderRetryIntervalMillis() != null) { + senderConfig.setRetryIntervalMs(getSenderRetryIntervalMillis()); + } - if (getSenderWorkBufSize() != null) { - senderConfig.setWorkBufSize(getSenderWorkBufSize()); - } + if (getSenderMaxRetryIntervalMillis() != null) { + senderConfig.setMaxRetryIntervalMs(getSenderMaxRetryIntervalMillis()); + } - return senderConfig; + if (getSenderRetryFactor() != null) { + senderConfig.setRetryFactor(getSenderRetryFactor()); } - @Override - public String toString() - { - return "FluencyBuilder{" + - "senderRetryMax=" + senderRetryMax + - ", senderRetryIntervalMillis=" + senderRetryIntervalMillis + - ", senderMaxRetryIntervalMillis=" + senderMaxRetryIntervalMillis + - ", senderRetryFactor=" + senderRetryFactor + - ", senderWorkBufSize=" + senderWorkBufSize + - "} " + super.toString(); + if (getErrorHandler() != null) { + senderConfig.setErrorHandler(getErrorHandler()); } - private MessagePackRecordFormatter buildRecordFormatter() - { - return new MessagePackRecordFormatter(); + if (getSenderWorkBufSize() != null) { + senderConfig.setWorkBufSize(getSenderWorkBufSize()); } - private Ingester buildIngester(TreasureDataSender.Config senderConfig) - { - TreasureDataSender sender = new TreasureDataSender(senderConfig); + return senderConfig; + } - return new TreasureDataIngester(sender); - } + @Override + public String toString() { + return "FluencyBuilder{" + + "senderRetryMax=" + + senderRetryMax + + ", senderRetryIntervalMillis=" + + senderRetryIntervalMillis + + ", senderMaxRetryIntervalMillis=" + + senderMaxRetryIntervalMillis + + ", senderRetryFactor=" + + senderRetryFactor + + ", senderWorkBufSize=" + + senderWorkBufSize + + "} " + + super.toString(); + } + + private MessagePackRecordFormatter buildRecordFormatter() { + return new MessagePackRecordFormatter(); + } + + private Ingester buildIngester(TreasureDataSender.Config senderConfig) { + TreasureDataSender sender = new TreasureDataSender(senderConfig); + + return new TreasureDataIngester(sender); + } } diff --git a/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/ingester/TreasureDataIngester.java b/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/ingester/TreasureDataIngester.java index 547f04ec..9bf4a261 100644 --- a/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/ingester/TreasureDataIngester.java +++ b/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/ingester/TreasureDataIngester.java @@ -16,43 +16,34 @@ package org.komamitsu.fluency.treasuredata.ingester; +import java.io.IOException; +import java.nio.ByteBuffer; import org.komamitsu.fluency.ingester.Ingester; import org.komamitsu.fluency.ingester.sender.Sender; import org.komamitsu.fluency.treasuredata.ingester.sender.TreasureDataSender; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.nio.ByteBuffer; - -public class TreasureDataIngester - implements Ingester -{ - private static final Logger LOG = LoggerFactory.getLogger(TreasureDataIngester.class); - private final TreasureDataSender sender; +public class TreasureDataIngester implements Ingester { + private static final Logger LOG = LoggerFactory.getLogger(TreasureDataIngester.class); + private final TreasureDataSender sender; - public TreasureDataIngester(TreasureDataSender sender) - { - this.sender = sender; - } + public TreasureDataIngester(TreasureDataSender sender) { + this.sender = sender; + } - @Override - public void ingest(String tag, ByteBuffer dataBuffer) - throws IOException - { - sender.send(tag, dataBuffer); - } + @Override + public void ingest(String tag, ByteBuffer dataBuffer) throws IOException { + sender.send(tag, dataBuffer); + } - @Override - public Sender getSender() - { - return sender; - } + @Override + public Sender getSender() { + return sender; + } - @Override - public void close() - throws IOException - { - sender.close(); - } + @Override + public void close() throws IOException { + sender.close(); + } } diff --git a/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/ingester/sender/TreasureDataSender.java b/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/ingester/sender/TreasureDataSender.java index 9457fa22..219e0256 100644 --- a/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/ingester/sender/TreasureDataSender.java +++ b/fluency-treasuredata/src/main/java/org/komamitsu/fluency/treasuredata/ingester/sender/TreasureDataSender.java @@ -21,19 +21,6 @@ import com.treasuredata.client.TDClientBuilder; import com.treasuredata.client.TDClientHttpException; import com.treasuredata.client.TDClientHttpNotFoundException; -import net.jodah.failsafe.Failsafe; -import net.jodah.failsafe.RetryPolicy; -import org.komamitsu.fluency.NonRetryableException; -import org.komamitsu.fluency.RetryableException; -import org.komamitsu.fluency.ingester.sender.ErrorHandler; -import org.komamitsu.fluency.ingester.sender.Sender; -import org.komamitsu.fluency.validation.Validatable; -import org.komamitsu.fluency.validation.annotation.DecimalMin; -import org.komamitsu.fluency.validation.annotation.Min; -import org.msgpack.core.annotations.VisibleForTesting; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.File; import java.io.IOException; @@ -48,388 +35,363 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.zip.GZIPOutputStream; +import net.jodah.failsafe.Failsafe; +import net.jodah.failsafe.RetryPolicy; +import org.komamitsu.fluency.NonRetryableException; +import org.komamitsu.fluency.RetryableException; +import org.komamitsu.fluency.ingester.sender.ErrorHandler; +import org.komamitsu.fluency.ingester.sender.Sender; +import org.komamitsu.fluency.validation.Validatable; +import org.komamitsu.fluency.validation.annotation.DecimalMin; +import org.komamitsu.fluency.validation.annotation.Min; +import org.msgpack.core.annotations.VisibleForTesting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class TreasureDataSender - implements Closeable, Sender -{ - private static final Logger LOG = LoggerFactory.getLogger(TreasureDataSender.class); - private static final int RETRY_COUNT_FOR_DB_CREATE_DELETE_CONFLICT = 4; - private static final int RETRY_INTERVAL_MS_FOR_DB_CREATE_DELETE_CONFLICT = 1000; - private final Config config; - private final TDClient client; - private final RetryPolicy retryPolicy; - - public TreasureDataSender() - { - this(new Config()); - } - - public TreasureDataSender(Config config) - { - config.validateValues(); - this.config = config; - this.retryPolicy = - new RetryPolicy(). - handleIf(ex -> { - if (ex == null) { - // Success. Shouldn't retry. - return false; - } - - ErrorHandler errorHandler = config.getErrorHandler(); - - if (errorHandler != null) { - errorHandler.handle(ex); - } - - if (ex instanceof InterruptedException || ex instanceof NonRetryableException) { - return false; - } - - return true; - }). - withBackoff( - getRetryInternalMs(), - getMaxRetryInternalMs(), - ChronoUnit.MILLIS, - getRetryFactor()). - withMaxRetries(getRetryMax()); - - this.client = buildClient(); +public class TreasureDataSender implements Closeable, Sender { + private static final Logger LOG = LoggerFactory.getLogger(TreasureDataSender.class); + private static final int RETRY_COUNT_FOR_DB_CREATE_DELETE_CONFLICT = 4; + private static final int RETRY_INTERVAL_MS_FOR_DB_CREATE_DELETE_CONFLICT = 1000; + private final Config config; + private final TDClient client; + private final RetryPolicy retryPolicy; + + public TreasureDataSender() { + this(new Config()); + } + + public TreasureDataSender(Config config) { + config.validateValues(); + this.config = config; + this.retryPolicy = + new RetryPolicy() + .handleIf( + ex -> { + if (ex == null) { + // Success. Shouldn't retry. + return false; + } + + ErrorHandler errorHandler = config.getErrorHandler(); + + if (errorHandler != null) { + errorHandler.handle(ex); + } + + if (ex instanceof InterruptedException || ex instanceof NonRetryableException) { + return false; + } + + return true; + }) + .withBackoff( + getRetryInternalMs(), getMaxRetryInternalMs(), ChronoUnit.MILLIS, getRetryFactor()) + .withMaxRetries(getRetryMax()); + + this.client = buildClient(); + } + + @VisibleForTesting + protected TDClient buildClient() { + URI uri; + try { + uri = new URI(config.getEndpoint()); + } catch (URISyntaxException e) { + throw new NonRetryableException( + String.format("Invalid endpoint. %s", config.getEndpoint()), e); } - @VisibleForTesting - protected TDClient buildClient() - { - URI uri; - try { - uri = new URI(config.getEndpoint()); - } - catch (URISyntaxException e) { - throw new NonRetryableException( - String.format("Invalid endpoint. %s", config.getEndpoint()), e); - } + String host = uri.getHost() != null ? uri.getHost() : config.getEndpoint(); - String host = uri.getHost() != null ? uri.getHost() : config.getEndpoint(); - - TDClientBuilder builder = new TDClientBuilder(false) - .setEndpoint(host) - .setApiKey(config.getApikey()) - .setRetryLimit(config.getRetryMax()) - .setRetryInitialIntervalMillis(config.getRetryIntervalMs()) - .setRetryMaxIntervalMillis(config.getMaxRetryIntervalMs()) - .setRetryMultiplier(config.getRetryFactor()); - - if (uri.getScheme() != null && uri.getScheme().equals("http")) { - builder.setUseSSL(false); - } + TDClientBuilder builder = + new TDClientBuilder(false) + .setEndpoint(host) + .setApiKey(config.getApikey()) + .setRetryLimit(config.getRetryMax()) + .setRetryInitialIntervalMillis(config.getRetryIntervalMs()) + .setRetryMaxIntervalMillis(config.getMaxRetryIntervalMs()) + .setRetryMultiplier(config.getRetryFactor()); - if (uri.getPort() > 0) { - builder.setPort(uri.getPort()); - } - - return builder.build(); - } - - public TDClient getClient() - { - return client; + if (uri.getScheme() != null && uri.getScheme().equals("http")) { + builder.setUseSSL(false); } - public int getRetryInternalMs() - { - return config.getRetryIntervalMs(); + if (uri.getPort() > 0) { + builder.setPort(uri.getPort()); } - public int getMaxRetryInternalMs() - { - return config.getMaxRetryIntervalMs(); + return builder.build(); + } + + public TDClient getClient() { + return client; + } + + public int getRetryInternalMs() { + return config.getRetryIntervalMs(); + } + + public int getMaxRetryInternalMs() { + return config.getMaxRetryIntervalMs(); + } + + public float getRetryFactor() { + return config.getRetryFactor(); + } + + public int getRetryMax() { + return config.getRetryMax(); + } + + public int getWorkBufSize() { + return config.getWorkBufSize(); + } + + private void copyStreams(InputStream in, OutputStream out) throws IOException { + byte[] buf = new byte[getWorkBufSize()]; + while (true) { + int readLen = in.read(buf); + if (readLen < 0) { + break; + } + out.write(buf, 0, readLen); } + } - public float getRetryFactor() - { - return config.getRetryFactor(); + private boolean checkDatabaseAndWaitIfNeeded(String database) { + if (client.existsDatabase(database)) { + return true; } - public int getRetryMax() - { - return config.getRetryMax(); - } + LOG.warn("The database could be just removed or invisible. Retrying.... database={}", database); - public int getWorkBufSize() - { - return config.getWorkBufSize(); + try { + TimeUnit.MILLISECONDS.sleep(RETRY_INTERVAL_MS_FOR_DB_CREATE_DELETE_CONFLICT); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new NonRetryableException( + String.format("Failed to create database. database=%s", database), e); } - private void copyStreams(InputStream in, OutputStream out) - throws IOException - { - byte[] buf = new byte[getWorkBufSize()]; - while (true) { - int readLen = in.read(buf); - if (readLen < 0) { - break; + return false; + } + + private void createDatabase(String database) { + for (int i = 0; i < RETRY_COUNT_FOR_DB_CREATE_DELETE_CONFLICT; i++) { + try { + client.createDatabase(database); + LOG.info("Created database. database={}", database); + return; + } catch (TDClientHttpException e) { + switch (e.getStatusCode()) { + case 409: + LOG.info("The database already exists. database={}", database); + if (checkDatabaseAndWaitIfNeeded(database)) { + return; } - out.write(buf, 0, readLen); + // Retrying... + break; + case 401: + case 403: + case 404: + throw new NonRetryableException( + String.format("Failed to create database. database=%s", database), e); } + } catch (NonRetryableException e) { + throw e; + } catch (Throwable e) { + throw new RetryableException( + String.format("Failed to create database. database=%s", database), e); + } } - private boolean checkDatabaseAndWaitIfNeeded(String database) - { - if (client.existsDatabase(database)) { - return true; - } - - LOG.warn("The database could be just removed or invisible. Retrying.... database={}", database); - - try { - TimeUnit.MILLISECONDS.sleep(RETRY_INTERVAL_MS_FOR_DB_CREATE_DELETE_CONFLICT); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); + // Retry over + throw new NonRetryableException( + String.format( + "It seems you don't have a proper permission on the database. database=%s", database)); + } + + private void createTable(String database, String table) { + while (true) { + try { + client.createTable(database, table); + LOG.info("Created table. database={}, table={}", database, table); + return; + } catch (TDClientHttpException e) { + switch (e.getStatusCode()) { + case 409: + LOG.info("The table already exists. database={}, table={}", database, table); + return; + case 401: + case 403: throw new NonRetryableException( - String.format("Failed to create database. database=%s", database), e); + String.format("Failed to create table. database=%s, table=%s", database, table), e); + case 404: + createDatabase(database); + // Retry to create the table + break; + default: + throw new RetryableException( + String.format("Failed to create table. database=%s, table=%s", database, table), e); } - - return false; + } catch (NonRetryableException e) { + throw e; + } catch (Throwable e) { + throw new RetryableException( + String.format("Failed to create table. database=%s, table=%s", database, table), e); + } + } + } + + private void importData(String database, String table, String uniqueId, File file) { + LOG.debug( + "Importing data to TD table: database={}, table={}, uniqueId={}, fileSize={}", + database, + table, + uniqueId, + file.length()); + + while (true) { + try { + client.importFile(database, table, file, uniqueId); + return; + } catch (TDClientHttpNotFoundException e) { + createTable(database, table); + // Retry to create the table + } catch (NonRetryableException e) { + throw e; + } catch (Throwable e) { + throw new RetryableException( + String.format("Failed to import data. database=%s, table=%s", database, table), e); + } } + } + + public void send(String dbAndTableTag, ByteBuffer dataBuffer) throws IOException { + String[] dbAndTable = dbAndTableTag.split("\\."); + // TODO: Validation + String database = dbAndTable[0]; + String table = dbAndTable[1]; + + File file = File.createTempFile("tmp-fluency-", ".msgpack.gz"); + try { + try (InputStream in = new ByteBufferBackedInputStream(dataBuffer); + OutputStream out = + new GZIPOutputStream( + Files.newOutputStream(file.toPath(), StandardOpenOption.WRITE))) { + + copyStreams(in, out); + } + + String uniqueId = UUID.randomUUID().toString(); + Failsafe.with(retryPolicy).run(() -> importData(database, table, uniqueId, file)); + } finally { + if (!file.delete()) { + LOG.warn("Failed to delete a temp file: {}", file.getAbsolutePath()); + } + } + } - private void createDatabase(String database) - { - for (int i = 0; i < RETRY_COUNT_FOR_DB_CREATE_DELETE_CONFLICT; i++) { - try { - client.createDatabase(database); - LOG.info("Created database. database={}", database); - return; - } - catch (TDClientHttpException e) { - switch (e.getStatusCode()) { - case 409: - LOG.info("The database already exists. database={}", database); - if (checkDatabaseAndWaitIfNeeded(database)) { - return; - } - // Retrying... - break; - case 401: - case 403: - case 404: - throw new NonRetryableException( - String.format("Failed to create database. database=%s", database), e); - } - } - catch (NonRetryableException e) { - throw e; - } - catch (Throwable e) { - throw new RetryableException( - String.format("Failed to create database. database=%s", database), e); - } - } + @Override + public void close() throws IOException {} - // Retry over - throw new NonRetryableException( - String.format("It seems you don't have a proper permission on the database. database=%s", database)); - } + public static class Config extends Sender.Config implements Validatable { + private String endpoint = "https://api-import.treasuredata.com"; + private String apikey; - private void createTable(String database, String table) - { - while (true) { - try { - client.createTable(database, table); - LOG.info("Created table. database={}, table={}", database, table); - return; - } - catch (TDClientHttpException e) { - switch (e.getStatusCode()) { - case 409: - LOG.info("The table already exists. database={}, table={}", database, table); - return; - case 401: - case 403: - throw new NonRetryableException( - String.format("Failed to create table. database=%s, table=%s", database, table), e); - case 404: - createDatabase(database); - // Retry to create the table - break; - default: - throw new RetryableException( - String.format("Failed to create table. database=%s, table=%s", database, table), e); - } - } - catch (NonRetryableException e) { - throw e; - } - catch (Throwable e) { - throw new RetryableException( - String.format("Failed to create table. database=%s, table=%s", database, table), e); - } - } - } + @Min(10) + private int retryIntervalMs = 1000; - private void importData(String database, String table, String uniqueId, File file) - { - LOG.debug("Importing data to TD table: database={}, table={}, uniqueId={}, fileSize={}", - database, table, uniqueId, file.length()); + @Min(10) + private int maxRetryIntervalMs = 30000; - while (true) { - try { - client.importFile(database, table, file, uniqueId); - return; - } - catch (TDClientHttpNotFoundException e) { - createTable(database, table); - // Retry to create the table - } - catch (NonRetryableException e) { - throw e; - } - catch (Throwable e) { - throw new RetryableException( - String.format("Failed to import data. database=%s, table=%s", database, table), e); - } - } - } + @DecimalMin("1.0") + private float retryFactor = 2; - public void send(String dbAndTableTag, ByteBuffer dataBuffer) - throws IOException - { - String[] dbAndTable = dbAndTableTag.split("\\."); - // TODO: Validation - String database = dbAndTable[0]; - String table = dbAndTable[1]; - - File file = File.createTempFile("tmp-fluency-", ".msgpack.gz"); - try { - try (InputStream in = new ByteBufferBackedInputStream(dataBuffer); - OutputStream out = new GZIPOutputStream( - Files.newOutputStream( - file.toPath(), - StandardOpenOption.WRITE))) { - - copyStreams(in, out); - } + @Min(0) + private int retryMax = 10; - String uniqueId = UUID.randomUUID().toString(); - Failsafe.with(retryPolicy).run(() -> importData(database, table, uniqueId, file)); - } - finally { - if (!file.delete()) { - LOG.warn("Failed to delete a temp file: {}", file.getAbsolutePath()); - } - } - } + @Min(1024) + private int workBufSize = 8192; - @Override - public void close() - throws IOException - { + public String getEndpoint() { + return endpoint; } - public static class Config - extends Sender.Config - implements Validatable - { - private String endpoint = "https://api-import.treasuredata.com"; - private String apikey; - @Min(10) - private int retryIntervalMs = 1000; - @Min(10) - private int maxRetryIntervalMs = 30000; - @DecimalMin("1.0") - private float retryFactor = 2; - @Min(0) - private int retryMax = 10; - @Min(1024) - private int workBufSize = 8192; - - public String getEndpoint() - { - return endpoint; - } - - public void setEndpoint(String endpoint) - { - this.endpoint = endpoint; - } + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } - public String getApikey() - { - return apikey; - } + public String getApikey() { + return apikey; + } - public void setApikey(String apikey) - { - this.apikey = apikey; - } + public void setApikey(String apikey) { + this.apikey = apikey; + } - public int getRetryIntervalMs() - { - return retryIntervalMs; - } + public int getRetryIntervalMs() { + return retryIntervalMs; + } - public void setRetryIntervalMs(int retryIntervalMs) - { - this.retryIntervalMs = retryIntervalMs; - } + public void setRetryIntervalMs(int retryIntervalMs) { + this.retryIntervalMs = retryIntervalMs; + } - public int getMaxRetryIntervalMs() - { - return maxRetryIntervalMs; - } + public int getMaxRetryIntervalMs() { + return maxRetryIntervalMs; + } - public void setMaxRetryIntervalMs(int maxRetryIntervalMs) - { - this.maxRetryIntervalMs = maxRetryIntervalMs; - } + public void setMaxRetryIntervalMs(int maxRetryIntervalMs) { + this.maxRetryIntervalMs = maxRetryIntervalMs; + } - public float getRetryFactor() - { - return retryFactor; - } + public float getRetryFactor() { + return retryFactor; + } - public void setRetryFactor(float retryFactor) - { - this.retryFactor = retryFactor; - } + public void setRetryFactor(float retryFactor) { + this.retryFactor = retryFactor; + } - public int getRetryMax() - { - return retryMax; - } + public int getRetryMax() { + return retryMax; + } - public void setRetryMax(int retryMax) - { - this.retryMax = retryMax; - } + public void setRetryMax(int retryMax) { + this.retryMax = retryMax; + } - public int getWorkBufSize() - { - return workBufSize; - } + public int getWorkBufSize() { + return workBufSize; + } - public void setWorkBufSize(int workBufSize) - { - this.workBufSize = workBufSize; - } + public void setWorkBufSize(int workBufSize) { + this.workBufSize = workBufSize; + } - @Override - public String toString() - { - return "Config{" + - "endpoint='" + endpoint + '\'' + - ", retryIntervalMs=" + retryIntervalMs + - ", maxRetryIntervalMs=" + maxRetryIntervalMs + - ", retryFactor=" + retryFactor + - ", retryMax=" + retryMax + - ", workBufSize=" + workBufSize + - "} " + super.toString(); - } + @Override + public String toString() { + return "Config{" + + "endpoint='" + + endpoint + + '\'' + + ", retryIntervalMs=" + + retryIntervalMs + + ", maxRetryIntervalMs=" + + maxRetryIntervalMs + + ", retryFactor=" + + retryFactor + + ", retryMax=" + + retryMax + + ", workBufSize=" + + workBufSize + + "} " + + super.toString(); + } - void validateValues() - { - validate(); - } + void validateValues() { + validate(); } + } } diff --git a/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/FluencyBuilderForTreasureDataTest.java b/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/FluencyBuilderForTreasureDataTest.java index 8bb67a04..65cce117 100644 --- a/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/FluencyBuilderForTreasureDataTest.java +++ b/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/FluencyBuilderForTreasureDataTest.java @@ -16,174 +16,167 @@ package org.komamitsu.fluency.treasuredata; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + import com.treasuredata.client.TDClient; import com.treasuredata.client.TDClientConfig; import com.treasuredata.client.TDHttpClient; +import java.io.IOException; +import java.lang.reflect.Field; import org.junit.jupiter.api.Test; import org.komamitsu.fluency.Fluency; import org.komamitsu.fluency.buffer.Buffer; import org.komamitsu.fluency.flusher.Flusher; import org.komamitsu.fluency.treasuredata.ingester.sender.TreasureDataSender; -import java.io.IOException; -import java.lang.reflect.Field; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class FluencyBuilderForTreasureDataTest -{ - private static final String APIKEY = "12345/1qaz2wsx3edc4rfv5tgb6yhn"; - - private void assertBuffer(Buffer buffer) - { - assertEquals(512 * 1024 * 1024L, buffer.getMaxBufferSize()); - assertNull(buffer.getFileBackupDir()); - assertEquals("packed_forward", buffer.bufferFormatType()); - assertEquals(2f, buffer.getChunkExpandRatio()); - assertEquals(64 * 1024 * 1024, buffer.getChunkRetentionSize()); - assertEquals(4 * 1024 * 1024, buffer.getChunkInitialSize()); - assertEquals(30000, buffer.getChunkRetentionTimeMillis()); - assertFalse(buffer.getJvmHeapBufferMode()); +class FluencyBuilderForTreasureDataTest { + private static final String APIKEY = "12345/1qaz2wsx3edc4rfv5tgb6yhn"; + + private void assertBuffer(Buffer buffer) { + assertEquals(512 * 1024 * 1024L, buffer.getMaxBufferSize()); + assertNull(buffer.getFileBackupDir()); + assertEquals("packed_forward", buffer.bufferFormatType()); + assertEquals(2f, buffer.getChunkExpandRatio()); + assertEquals(64 * 1024 * 1024, buffer.getChunkRetentionSize()); + assertEquals(4 * 1024 * 1024, buffer.getChunkInitialSize()); + assertEquals(30000, buffer.getChunkRetentionTimeMillis()); + assertFalse(buffer.getJvmHeapBufferMode()); + } + + private void assertFlusher(Flusher flusher) { + assertFalse(flusher.isTerminated()); + assertEquals(600, flusher.getFlushAttemptIntervalMillis()); + assertEquals(60, flusher.getWaitUntilBufferFlushed()); + assertEquals(60, flusher.getWaitUntilTerminated()); + } + + private void assertDefaultFluentdSender( + TreasureDataSender sender, String expectedEndpoint, boolean expectedUseSsl) + throws NoSuchFieldException, IllegalAccessException { + assertEquals(1000, sender.getRetryInternalMs()); + assertEquals(30000, sender.getMaxRetryInternalMs()); + assertEquals(2.0f, sender.getRetryFactor()); + assertEquals(10, sender.getRetryMax()); + assertEquals(8192, sender.getWorkBufSize()); + + Field httpClientField = TDClient.class.getDeclaredField("httpClient"); + httpClientField.setAccessible(true); + TDHttpClient tdHttpClient = (TDHttpClient) httpClientField.get(sender.getClient()); + + Field configField = TDHttpClient.class.getDeclaredField("config"); + configField.setAccessible(true); + TDClientConfig config = (TDClientConfig) configField.get(tdHttpClient); + + assertEquals(expectedEndpoint, config.endpoint); + assertEquals(expectedUseSsl, config.useSSL); + assertEquals(FluencyBuilderForTreasureDataTest.APIKEY, config.apiKey.get()); + } + + @Test + void build() throws IOException, NoSuchFieldException, IllegalAccessException { + try (Fluency fluency = new FluencyBuilderForTreasureData().build(APIKEY)) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (TreasureDataSender) fluency.getFlusher().getIngester().getSender(), + "api-import.treasuredata.com", + true); } - - private void assertFlusher(Flusher flusher) - { - assertFalse(flusher.isTerminated()); - assertEquals(600, flusher.getFlushAttemptIntervalMillis()); - assertEquals(60, flusher.getWaitUntilBufferFlushed()); - assertEquals(60, flusher.getWaitUntilTerminated()); + } + + @Test + void buildWithCustomHttpsEndpoint() + throws IOException, NoSuchFieldException, IllegalAccessException { + try (Fluency fluency = + new FluencyBuilderForTreasureData().build(APIKEY, "https://custom.endpoint.org")) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (TreasureDataSender) fluency.getFlusher().getIngester().getSender(), + "custom.endpoint.org", + true); } - - private void assertDefaultFluentdSender( - TreasureDataSender sender, - String expectedEndpoint, - boolean expectedUseSsl) - throws NoSuchFieldException, IllegalAccessException - { - assertEquals(1000, sender.getRetryInternalMs()); - assertEquals(30000, sender.getMaxRetryInternalMs()); - assertEquals(2.0f, sender.getRetryFactor()); - assertEquals(10, sender.getRetryMax()); - assertEquals(8192, sender.getWorkBufSize()); - - Field httpClientField = TDClient.class.getDeclaredField("httpClient"); - httpClientField.setAccessible(true); - TDHttpClient tdHttpClient = (TDHttpClient) httpClientField.get(sender.getClient()); - - Field configField = TDHttpClient.class.getDeclaredField("config"); - configField.setAccessible(true); - TDClientConfig config = (TDClientConfig) configField.get(tdHttpClient); - - assertEquals(expectedEndpoint, config.endpoint); - assertEquals(expectedUseSsl, config.useSSL); - assertEquals(FluencyBuilderForTreasureDataTest.APIKEY, config.apiKey.get()); + } + + @Test + void buildWithCustomHttpsEndpointWithoutScheme() + throws IOException, NoSuchFieldException, IllegalAccessException { + try (Fluency fluency = + new FluencyBuilderForTreasureData().build(APIKEY, "custom.endpoint.org")) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (TreasureDataSender) fluency.getFlusher().getIngester().getSender(), + "custom.endpoint.org", + true); } - - @Test - void build() - throws IOException, NoSuchFieldException, IllegalAccessException - { - try (Fluency fluency = new FluencyBuilderForTreasureData().build(APIKEY)) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (TreasureDataSender) fluency.getFlusher().getIngester().getSender(), - "api-import.treasuredata.com", true); - } + } + + @Test + void buildWithCustomHttpEndpoint() + throws IOException, NoSuchFieldException, IllegalAccessException { + try (Fluency fluency = + new FluencyBuilderForTreasureData().build(APIKEY, "http://custom.endpoint.org")) { + assertBuffer(fluency.getBuffer()); + assertFlusher(fluency.getFlusher()); + assertDefaultFluentdSender( + (TreasureDataSender) fluency.getFlusher().getIngester().getSender(), + "custom.endpoint.org", + false); } - - @Test - void buildWithCustomHttpsEndpoint() - throws IOException, NoSuchFieldException, IllegalAccessException - { - try (Fluency fluency = new FluencyBuilderForTreasureData().build(APIKEY, "https://custom.endpoint.org")) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (TreasureDataSender) fluency.getFlusher().getIngester().getSender(), - "custom.endpoint.org", true); - } - } - - @Test - void buildWithCustomHttpsEndpointWithoutScheme() - throws IOException, NoSuchFieldException, IllegalAccessException - { - try (Fluency fluency = new FluencyBuilderForTreasureData().build(APIKEY, "custom.endpoint.org")) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (TreasureDataSender) fluency.getFlusher().getIngester().getSender(), - "custom.endpoint.org", true); - } - } - - @Test - void buildWithCustomHttpEndpoint() - throws IOException, NoSuchFieldException, IllegalAccessException - { - try (Fluency fluency = new FluencyBuilderForTreasureData().build(APIKEY, "http://custom.endpoint.org")) { - assertBuffer(fluency.getBuffer()); - assertFlusher(fluency.getFlusher()); - assertDefaultFluentdSender( - (TreasureDataSender) fluency.getFlusher().getIngester().getSender(), - "custom.endpoint.org", false); - } - } - - @Test - void buildWithAllCustomConfig() - throws IOException - { - String tmpdir = System.getProperty("java.io.tmpdir"); - assertNotNull(tmpdir); - - FluencyBuilderForTreasureData builder = new FluencyBuilderForTreasureData(); - builder.setFlushAttemptIntervalMillis(200); - builder.setMaxBufferSize(Long.MAX_VALUE); - builder.setBufferChunkInitialSize(7 * 1024 * 1024); - builder.setBufferChunkRetentionSize(13 * 1024 * 1024); - builder.setBufferChunkRetentionTimeMillis(19 * 1000); - builder.setJvmHeapBufferMode(true); - builder.setWaitUntilBufferFlushed(42); - builder.setWaitUntilFlusherTerminated(24); - builder.setFileBackupDir(tmpdir); - builder.setSenderRetryIntervalMillis(1234); - builder.setSenderMaxRetryIntervalMillis(345678); - builder.setSenderRetryFactor(3.14f); - builder.setSenderRetryMax(17); - builder.setSenderWorkBufSize(123456); - ; - - try (Fluency fluency = builder.build(APIKEY)) { - assertEquals(Buffer.class, fluency.getBuffer().getClass()); - Buffer buffer = fluency.getBuffer(); - assertEquals(Long.MAX_VALUE, buffer.getMaxBufferSize()); - assertEquals(tmpdir, buffer.getFileBackupDir()); - assertEquals("packed_forward", buffer.bufferFormatType()); - assertEquals(19 * 1000, buffer.getChunkRetentionTimeMillis()); - assertEquals(2f, buffer.getChunkExpandRatio()); - assertEquals(7 * 1024 * 1024, buffer.getChunkInitialSize()); - assertEquals(13 * 1024 * 1024, buffer.getChunkRetentionSize()); - assertTrue(buffer.getJvmHeapBufferMode()); - - Flusher flusher = fluency.getFlusher(); - assertFalse(flusher.isTerminated()); - assertEquals(200, flusher.getFlushAttemptIntervalMillis()); - assertEquals(42, flusher.getWaitUntilBufferFlushed()); - assertEquals(24, flusher.getWaitUntilTerminated()); - - assertEquals(TreasureDataSender.class, flusher.getIngester().getSender().getClass()); - TreasureDataSender sender = (TreasureDataSender) flusher.getIngester().getSender(); - assertEquals(1234, sender.getRetryInternalMs()); - assertEquals(345678, sender.getMaxRetryInternalMs()); - assertEquals(3.14f, sender.getRetryFactor()); - assertEquals(17, sender.getRetryMax()); - assertEquals(123456, sender.getWorkBufSize()); - } + } + + @Test + void buildWithAllCustomConfig() throws IOException { + String tmpdir = System.getProperty("java.io.tmpdir"); + assertNotNull(tmpdir); + + FluencyBuilderForTreasureData builder = new FluencyBuilderForTreasureData(); + builder.setFlushAttemptIntervalMillis(200); + builder.setMaxBufferSize(Long.MAX_VALUE); + builder.setBufferChunkInitialSize(7 * 1024 * 1024); + builder.setBufferChunkRetentionSize(13 * 1024 * 1024); + builder.setBufferChunkRetentionTimeMillis(19 * 1000); + builder.setJvmHeapBufferMode(true); + builder.setWaitUntilBufferFlushed(42); + builder.setWaitUntilFlusherTerminated(24); + builder.setFileBackupDir(tmpdir); + builder.setSenderRetryIntervalMillis(1234); + builder.setSenderMaxRetryIntervalMillis(345678); + builder.setSenderRetryFactor(3.14f); + builder.setSenderRetryMax(17); + builder.setSenderWorkBufSize(123456); + ; + + try (Fluency fluency = builder.build(APIKEY)) { + assertEquals(Buffer.class, fluency.getBuffer().getClass()); + Buffer buffer = fluency.getBuffer(); + assertEquals(Long.MAX_VALUE, buffer.getMaxBufferSize()); + assertEquals(tmpdir, buffer.getFileBackupDir()); + assertEquals("packed_forward", buffer.bufferFormatType()); + assertEquals(19 * 1000, buffer.getChunkRetentionTimeMillis()); + assertEquals(2f, buffer.getChunkExpandRatio()); + assertEquals(7 * 1024 * 1024, buffer.getChunkInitialSize()); + assertEquals(13 * 1024 * 1024, buffer.getChunkRetentionSize()); + assertTrue(buffer.getJvmHeapBufferMode()); + + Flusher flusher = fluency.getFlusher(); + assertFalse(flusher.isTerminated()); + assertEquals(200, flusher.getFlushAttemptIntervalMillis()); + assertEquals(42, flusher.getWaitUntilBufferFlushed()); + assertEquals(24, flusher.getWaitUntilTerminated()); + + assertEquals(TreasureDataSender.class, flusher.getIngester().getSender().getClass()); + TreasureDataSender sender = (TreasureDataSender) flusher.getIngester().getSender(); + assertEquals(1234, sender.getRetryInternalMs()); + assertEquals(345678, sender.getMaxRetryInternalMs()); + assertEquals(3.14f, sender.getRetryFactor()); + assertEquals(17, sender.getRetryMax()); + assertEquals(123456, sender.getWorkBufSize()); } + } } diff --git a/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/ingester/TreasureDataIngesterTest.java b/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/ingester/TreasureDataIngesterTest.java index 3d8c1fa4..7b45545f 100644 --- a/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/ingester/TreasureDataIngesterTest.java +++ b/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/ingester/TreasureDataIngesterTest.java @@ -16,15 +16,6 @@ package org.komamitsu.fluency.treasuredata.ingester; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.treasuredata.ingester.sender.TreasureDataSender; -import org.mockito.ArgumentCaptor; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; - import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.eq; @@ -32,47 +23,46 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -class TreasureDataIngesterTest -{ - private static final Charset CHARSET = Charset.forName("UTF-8"); - private static final String TAG = "foo.bar"; - private static final byte[] DATA = "hello, world".getBytes(CHARSET); - private TreasureDataSender treasureDataSender; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.treasuredata.ingester.sender.TreasureDataSender; +import org.mockito.ArgumentCaptor; + +class TreasureDataIngesterTest { + private static final Charset CHARSET = Charset.forName("UTF-8"); + private static final String TAG = "foo.bar"; + private static final byte[] DATA = "hello, world".getBytes(CHARSET); + private TreasureDataSender treasureDataSender; - @BeforeEach - void setUp() - throws Exception - { - treasureDataSender = mock(TreasureDataSender.class); - } + @BeforeEach + void setUp() throws Exception { + treasureDataSender = mock(TreasureDataSender.class); + } - @Test - void ingest() - throws IOException - { - TreasureDataIngester ingester = new TreasureDataIngester(treasureDataSender); - ingester.ingest(TAG, ByteBuffer.wrap(DATA)); - ArgumentCaptor byteBufferArgumentCaptor = ArgumentCaptor.forClass(ByteBuffer.class); - verify(treasureDataSender, times(1)).send(eq(TAG), byteBufferArgumentCaptor.capture()); + @Test + void ingest() throws IOException { + TreasureDataIngester ingester = new TreasureDataIngester(treasureDataSender); + ingester.ingest(TAG, ByteBuffer.wrap(DATA)); + ArgumentCaptor byteBufferArgumentCaptor = ArgumentCaptor.forClass(ByteBuffer.class); + verify(treasureDataSender, times(1)).send(eq(TAG), byteBufferArgumentCaptor.capture()); - assertEquals(1, byteBufferArgumentCaptor.getAllValues().size()); - assertArrayEquals(DATA, byteBufferArgumentCaptor.getAllValues().get(0).array()); - } + assertEquals(1, byteBufferArgumentCaptor.getAllValues().size()); + assertArrayEquals(DATA, byteBufferArgumentCaptor.getAllValues().get(0).array()); + } - @Test - void getSender() - { - assertEquals(treasureDataSender, - new TreasureDataIngester(treasureDataSender).getSender()); - } + @Test + void getSender() { + assertEquals(treasureDataSender, new TreasureDataIngester(treasureDataSender).getSender()); + } - @Test - void close() - throws IOException - { - TreasureDataIngester ingester = new TreasureDataIngester(treasureDataSender); - ingester.close(); + @Test + void close() throws IOException { + TreasureDataIngester ingester = new TreasureDataIngester(treasureDataSender); + ingester.close(); - verify(treasureDataSender, times(1)).close(); - } + verify(treasureDataSender, times(1)).close(); + } } diff --git a/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/ingester/sender/TreasureDataSenderTest.java b/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/ingester/sender/TreasureDataSenderTest.java index ed1893f1..b94927eb 100644 --- a/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/ingester/sender/TreasureDataSenderTest.java +++ b/fluency-treasuredata/src/test/java/org/komamitsu/fluency/treasuredata/ingester/sender/TreasureDataSenderTest.java @@ -16,24 +16,6 @@ package org.komamitsu.fluency.treasuredata.ingester.sender; -import com.treasuredata.client.TDClient; -import com.treasuredata.client.TDClientHttpConflictException; -import com.treasuredata.client.TDClientHttpNotFoundException; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.komamitsu.fluency.NonRetryableException; -import org.mockito.ArgumentCaptor; - -import java.io.DataInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.Charset; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.zip.GZIPInputStream; - import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -48,174 +30,191 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -class TreasureDataSenderTest -{ - private final static Charset CHARSET = Charset.forName("UTF-8"); - private final static String DB = "foodb"; - private final static String TABLE = "bartbl"; - private final static String DB_AND_TABLE = "foodb.bartbl"; - private final static byte[] DATA = "hello, world".getBytes(CHARSET); - private TDClient client; - private TreasureDataSender sender; - - @BeforeEach - void setUp() - { - client = mock(TDClient.class); - sender = new TreasureDataSender(new TreasureDataSender.Config()) - { - @Override - protected TDClient buildClient() - { - return client; - } +import com.treasuredata.client.TDClient; +import com.treasuredata.client.TDClientHttpConflictException; +import com.treasuredata.client.TDClientHttpNotFoundException; +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.GZIPInputStream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.komamitsu.fluency.NonRetryableException; +import org.mockito.ArgumentCaptor; + +class TreasureDataSenderTest { + private static final Charset CHARSET = Charset.forName("UTF-8"); + private static final String DB = "foodb"; + private static final String TABLE = "bartbl"; + private static final String DB_AND_TABLE = "foodb.bartbl"; + private static final byte[] DATA = "hello, world".getBytes(CHARSET); + private TDClient client; + private TreasureDataSender sender; + + @BeforeEach + void setUp() { + client = mock(TDClient.class); + sender = + new TreasureDataSender(new TreasureDataSender.Config()) { + @Override + protected TDClient buildClient() { + return client; + } }; + } + + private void assertImportedFile(File file) throws IOException { + try (FileInputStream fileInputStream = new FileInputStream(file); + GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream); + DataInputStream dataInputStream = new DataInputStream(gzipInputStream)) { + byte[] data = new byte[DATA.length]; + dataInputStream.readFully(data); + assertArrayEquals(DATA, data); } - - private void assertImportedFile(File file) - throws IOException - { - try (FileInputStream fileInputStream = new FileInputStream(file); - GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream); - DataInputStream dataInputStream = new DataInputStream(gzipInputStream)) { - byte[] data = new byte[DATA.length]; - dataInputStream.readFully(data); - assertArrayEquals(DATA, data); - } + } + + @Test + void send() throws IOException { + doAnswer( + invocation -> { + assertImportedFile(invocation.getArgument(2)); + return null; + }) + .when(client) + .importFile(anyString(), anyString(), any(File.class), anyString()); + + sender.send(DB_AND_TABLE, ByteBuffer.wrap(DATA)); + ArgumentCaptor uniqueIdArgumentCaptor = ArgumentCaptor.forClass(String.class); + verify(client, times(1)) + .importFile(eq(DB), eq(TABLE), any(File.class), uniqueIdArgumentCaptor.capture()); + verify(client, times(0)).createDatabase(anyString()); + verify(client, times(0)).createTable(anyString(), anyString()); + UUID.fromString(uniqueIdArgumentCaptor.getValue()); + } + + @Test + void sendWithCreatingTable() throws IOException { + AtomicInteger importToTableCalls = new AtomicInteger(); + doAnswer( + invocation -> { + if (importToTableCalls.getAndIncrement() == 0) { + throw new TDClientHttpNotFoundException("Not Found!!!!"); + } + assertImportedFile(invocation.getArgument(2)); + return null; + }) + .when(client) + .importFile(anyString(), anyString(), any(File.class), anyString()); + + sender.send(DB_AND_TABLE, ByteBuffer.wrap(DATA)); + ArgumentCaptor uniqueIdArgumentCaptor = ArgumentCaptor.forClass(String.class); + verify(client, times(2)) + .importFile(eq(DB), eq(TABLE), any(File.class), uniqueIdArgumentCaptor.capture()); + verify(client, times(0)).createDatabase(anyString()); + verify(client, times(1)).createTable(eq(DB), eq(TABLE)); + UUID.fromString(uniqueIdArgumentCaptor.getValue()); + } + + @Test + void sendWithCreatingDatabase() throws IOException { + AtomicInteger importToTableCalls = new AtomicInteger(); + doAnswer( + invocation -> { + if (importToTableCalls.getAndIncrement() == 0) { + throw new TDClientHttpNotFoundException("Not Found!!!!"); + } + assertImportedFile(invocation.getArgument(2)); + return null; + }) + .when(client) + .importFile(anyString(), anyString(), any(File.class), anyString()); + + AtomicInteger createTableCalls = new AtomicInteger(); + doAnswer( + invocation -> { + if (createTableCalls.getAndIncrement() == 0) { + throw new TDClientHttpNotFoundException("Not Found!!!!"); + } + return null; + }) + .when(client) + .createTable(anyString(), anyString()); + + sender.send(DB_AND_TABLE, ByteBuffer.wrap(DATA)); + ArgumentCaptor uniqueIdArgumentCaptor = ArgumentCaptor.forClass(String.class); + verify(client, times(2)) + .importFile(eq(DB), eq(TABLE), any(File.class), uniqueIdArgumentCaptor.capture()); + verify(client, times(1)).createDatabase(eq(DB)); + verify(client, times(2)).createTable(eq(DB), eq(TABLE)); + UUID.fromString(uniqueIdArgumentCaptor.getValue()); + } + + @Test + public void sendWithLackOfPermissionOnDatabase() throws IOException { + doThrow(new TDClientHttpNotFoundException("Not Found!!!!")) + .when(client) + .importFile(anyString(), anyString(), any(File.class), anyString()); + + doThrow(new TDClientHttpNotFoundException("Not Found!!!!")) + .when(client) + .createTable(anyString(), anyString()); + + doThrow(new TDClientHttpConflictException("Conflict!!!!")) + .when(client) + .createDatabase(anyString()); + + doReturn(false).when(client).existsDatabase(anyString()); + + try { + sender.send(DB_AND_TABLE, ByteBuffer.wrap(DATA)); + fail(); + } catch (NonRetryableException e) { + assertTrue(true); } - - @Test - void send() - throws IOException + ArgumentCaptor uniqueIdArgumentCaptor = ArgumentCaptor.forClass(String.class); + verify(client, times(1)) + .importFile(eq(DB), eq(TABLE), any(File.class), uniqueIdArgumentCaptor.capture()); + verify(client, times(4)).createDatabase(eq(DB)); + verify(client, times(4)).existsDatabase(eq(DB)); + verify(client, times(1)).createTable(eq(DB), eq(TABLE)); + UUID.fromString(uniqueIdArgumentCaptor.getValue()); + } + + @Test + void validateConfig() { { - doAnswer(invocation -> { - assertImportedFile(invocation.getArgument(2)); - return null; - }).when(client).importFile(anyString(), anyString(), any(File.class), anyString()); - - sender.send(DB_AND_TABLE, ByteBuffer.wrap(DATA)); - ArgumentCaptor uniqueIdArgumentCaptor = ArgumentCaptor.forClass(String.class); - verify(client, times(1)).importFile( - eq(DB), eq(TABLE), any(File.class), uniqueIdArgumentCaptor.capture()); - verify(client, times(0)).createDatabase(anyString()); - verify(client, times(0)).createTable(anyString(), anyString()); - UUID.fromString(uniqueIdArgumentCaptor.getValue()); + TreasureDataSender.Config config = new TreasureDataSender.Config(); + config.setRetryIntervalMs(9); + assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); } - @Test - void sendWithCreatingTable() - throws IOException { - AtomicInteger importToTableCalls = new AtomicInteger(); - doAnswer(invocation -> { - if (importToTableCalls.getAndIncrement() == 0) { - throw new TDClientHttpNotFoundException("Not Found!!!!"); - } - assertImportedFile(invocation.getArgument(2)); - return null; - }).when(client).importFile(anyString(), anyString(), any(File.class), anyString()); - - sender.send(DB_AND_TABLE, ByteBuffer.wrap(DATA)); - ArgumentCaptor uniqueIdArgumentCaptor = ArgumentCaptor.forClass(String.class); - verify(client, times(2)).importFile( - eq(DB), eq(TABLE), any(File.class), uniqueIdArgumentCaptor.capture()); - verify(client, times(0)).createDatabase(anyString()); - verify(client, times(1)).createTable(eq(DB), eq(TABLE)); - UUID.fromString(uniqueIdArgumentCaptor.getValue()); + TreasureDataSender.Config config = new TreasureDataSender.Config(); + config.setMaxRetryIntervalMs(9); + assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); } - @Test - void sendWithCreatingDatabase() - throws IOException { - AtomicInteger importToTableCalls = new AtomicInteger(); - doAnswer(invocation -> { - if (importToTableCalls.getAndIncrement() == 0) { - throw new TDClientHttpNotFoundException("Not Found!!!!"); - } - assertImportedFile(invocation.getArgument(2)); - return null; - }).when(client).importFile(anyString(), anyString(), any(File.class), anyString()); - - AtomicInteger createTableCalls = new AtomicInteger(); - doAnswer(invocation -> { - if (createTableCalls.getAndIncrement() == 0) { - throw new TDClientHttpNotFoundException("Not Found!!!!"); - } - return null; - }).when(client).createTable(anyString(), anyString()); - - sender.send(DB_AND_TABLE, ByteBuffer.wrap(DATA)); - ArgumentCaptor uniqueIdArgumentCaptor = ArgumentCaptor.forClass(String.class); - verify(client, times(2)).importFile( - eq(DB), eq(TABLE), any(File.class), uniqueIdArgumentCaptor.capture()); - verify(client, times(1)).createDatabase(eq(DB)); - verify(client, times(2)).createTable(eq(DB), eq(TABLE)); - UUID.fromString(uniqueIdArgumentCaptor.getValue()); + TreasureDataSender.Config config = new TreasureDataSender.Config(); + config.setRetryMax(-1); + assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); } - @Test - public void sendWithLackOfPermissionOnDatabase() - throws IOException { - doThrow(new TDClientHttpNotFoundException("Not Found!!!!")) - .when(client).importFile(anyString(), anyString(), any(File.class), anyString()); - - doThrow(new TDClientHttpNotFoundException("Not Found!!!!")) - .when(client).createTable(anyString(), anyString()); - - doThrow(new TDClientHttpConflictException("Conflict!!!!")) - .when(client).createDatabase(anyString()); - - doReturn(false).when(client).existsDatabase(anyString()); - - try { - sender.send(DB_AND_TABLE, ByteBuffer.wrap(DATA)); - fail(); - } - catch (NonRetryableException e) { - assertTrue(true); - } - ArgumentCaptor uniqueIdArgumentCaptor = ArgumentCaptor.forClass(String.class); - verify(client, times(1)).importFile( - eq(DB), eq(TABLE), any(File.class), uniqueIdArgumentCaptor.capture()); - verify(client, times(4)).createDatabase(eq(DB)); - verify(client, times(4)).existsDatabase(eq(DB)); - verify(client, times(1)).createTable(eq(DB), eq(TABLE)); - UUID.fromString(uniqueIdArgumentCaptor.getValue()); + TreasureDataSender.Config config = new TreasureDataSender.Config(); + config.setRetryFactor(0.9f); + assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); } - @Test - void validateConfig() { - { - TreasureDataSender.Config config = new TreasureDataSender.Config(); - config.setRetryIntervalMs(9); - assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); - } - - { - TreasureDataSender.Config config = new TreasureDataSender.Config(); - config.setMaxRetryIntervalMs(9); - assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); - } - - { - TreasureDataSender.Config config = new TreasureDataSender.Config(); - config.setRetryMax(-1); - assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); - } - - { - TreasureDataSender.Config config = new TreasureDataSender.Config(); - config.setRetryFactor(0.9f); - assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); - } - - { - TreasureDataSender.Config config = new TreasureDataSender.Config(); - config.setWorkBufSize(1023); - assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); - } + TreasureDataSender.Config config = new TreasureDataSender.Config(); + config.setWorkBufSize(1023); + assertThrows(IllegalArgumentException.class, () -> new TreasureDataSender(config)); } -} \ No newline at end of file + } +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index e708b1c023ec8b20f512888fe07c5bd3ff77bb8f..7f93135c49b765f8051ef9d0a6055ff8e46073d8 100644 GIT binary patch delta 44866 zcmZ6yV~{3M)92l`t*d+5wmogzwr%${jcMDqZQHiZY1^EgclU{V_kKTAoQ$fhi29Iu z@>iMvGdKf&b_WirC<6|Gk*MT`8IOk!ijf$wgc~oR`Oo-w2N?nc1jNDFf)O0#|9Y)s z{-1}55TF2T3=j|)n14<}AR(u#9+Yq(Ao|!KAT){jCRmAAYuLbSO=w^IWx%&S-N(_x zu*i%umUPxo11kb-zz{5K%+(qcIZ{gEQgDLqWh6bxS=J)8yrq>4cDCz0sOy3dXTAtW z8|cOYsGU{54|2y#PSUfFM?;mWY&Xuph_swB`6Qom&H7|#Cr5YxX)8BD+UVA(U8sP*+u8?shKi4yhV*4yh93bXZq z1G2SJ^C)n)>_E=5(ewkWy-SV3LB$DrhauJD^-f-Jr}#j=OXqi=4@Q?p|A%T-|A!je z9ypvpS%FeR2e=0{*U9KMO~xHDCeet*(W=C8Li}VK5jxO+tFQp=W#X$SB6hzk;={2w z0{YeoGq;Ztldo(~g+}|%3X}7I)!$bgjqhl55Ke$nP>>P1H}3$|w;s-5pb(BQG0h-_ z4)zyW#vh#g|ha_79S)BJH#{K2`f^Wkouea2r_va7i zY=3Qs@C)GheoC3$aKa!j#Gls8@uX_XD-mx#JD6bUFdHH0Jp%CR#GL>v+MIo z-Xwe;xhN*D$#`0~Odj2qt7PA*J(7~P99at6b1C{LVx@r~A!kv-pFBdrA|2-TSunq6 zNjma624n3(`oRoC+-W7Sq=C_(j;3vQg)=BCW> z;2eU45kg1Du`=g5*>|sk-L0AKn~h_~GE(N?r&XrDJ0v%o0~PhbP1f@vKP_5|10I zs5-!ov1UfQ(F6KLN2}7wiyz5X9OzArB=$(iGG?h>V#Ego2{jb(XM83IC3cCgHeI*K z*v8|cMS}7}A&V&ta%h$r0ag4&$xYwHE_Aq@Ps9Pq`lkGVbr;;78P=eX3l7L#Gqh6? zk>P7|(Se;9YT3|%?5V2ceG*Nl?SrDvRewybjPbMEuu2SpJ2%mW7UQ$k`*uJ5FwjM2iN*lz#sVY zFSZ9L#g#4(qKMFvHnyEf1_EbHojThmilFjsTL{r1g z{>kg_Km+SnO#f-R4EY&*e)fSM)~|^Em3tm|{;`6VT;V6yT+z&kPJ9aeMC_liz?EC< z&t$zpHN&V1y|O(AyzdyGB=on6VL4FaZtM*u7Q4gc{Pvf9Y*Vqacp$Z@6vf|!Np!Gi z!ZDvUY84L71+OWA)ehjH6ezu5xn-_QyH&@!bjx3*Fsk6R(r^}W?N`xu3K~)F%tdxt zG@Iz3Y69Vk!MUyl%BU3-6~QH8Zy9A+>=9+lqS-K11^VdlL&6lR2*;wWvXnqc10JhU zH1TW2dxN6zVP*JTSen{!auTY#N<9S%IbJr?xiN@CT)ZSPE@|Lz`80K^i$s)C+0$S% z3gIAvu+gU^RBPQ#>n+u|r0&sLRwz3;I?TzUw&MuR{P5%hFtK2^dbribt0~>93{Cn8 z%=x@qG7xO`*h^W(Y4jZm)ri5qPisM&xc{xgolZ}KB4eO{Z z$m1D5!*nZp71hC*PuC1YKmOjOrX z(G|x!A=th#)?@$qx`m*fp%4KBVWd52TbpUl%o&T43}Z9)4&_Sd-&FkH8Xxz<$<-5e zFZj=CR@s*pCQOShE~94jO`#TP-o>~|aA)y^T|f9Xvky0LkEc8V_dN8LXE7kdj(n4j z1b5PA3aHm}w|ozH;4#1~CB#+zwEy2ukt&PE;v#>%HC`LnB!fu1da*Vy z&GgMiloIKh+wHtD(Ya$at;VSNHU>sE^@!Ka0Tzd3? z*iLZrhC;_a(KS`OMO-$qw0r$7vR$n&(iCnA#+Dcl)7OF^(vXKZX8GbLET9?E$R%@KjV{uDz>V{wrh^&O8HVMSbs@U#lL-IJsA+5fc>%X>z_ugEd~RXgp)Z zWgkqvlQe2>m*qn1;O-eEe<2H9N-#noIdUSaZ5brDuD0DwZmv2KO>eFa%sL2=w-J&Q z{q7CyJ)S;H^JoonPP6$9j<9c~%Z?2S-j>mu%oS)8CT0x014+(MSTjMVimb~0+j<#B|wgOd@CZa7tCijsPTvRY<5ixOcu+7%rR zHg&@;YsL<*{lZ)ei>!y;Zi@qL3(MIEG+BcKPu)Xk*e{!0ru8}Ybct6NbjzsfNPMI6VMWX+Etux0FQ3?au}C*h%QHrw_}eykN8CG zVGJ<7!gths*nawtF=kND_?Jfw6{*hyOT)w+&e(x%jF=9VL;X^st@i8Rfj+T-X} z(c$6C0xOoCvoG#mYy}+(s@q+79RMZ7JTkyQybPh!U z0gatF&+muyceQ`D)bInnOx4k(Od?-VO`y0D3Ylviq{)(!WcrLc|Lc;My)_Rp;OXWDtDLV#j4Li#0x{#!Er4gM{O z(>u707{BPC{+U7{=Scq-a)-i+O%l07m4F0mSX5M)H({7(klsQNbY;Js5UQfoobJZbVF z=PB@v(X>&MQ6vU}K)3dBj@oG12GO|0xvrg}q;L0nAYhTX2%V!?+0ts4gnx|8Wdj$Y zb=fS&Y`n-brYq;A{9II4R+KAB=qAL4m=jF-E37NLa*aAx*FPQLWxCYOQxOgU`1qBM%0tl{*hY9S!i_j*WysEXN27jWEgzAOD!>$Ef?>f!qAY3L)g)hQ;8by z(xsv@Wj`8Yb|gR=J{u?0CUjv>jt4TaU1*OaA+mOxRdkFnkI$Lc+G!NA!Cr_tSnK)I zOFT5BM4HQrEIW@&OUKisE0 zQF9S8+h$2&agE}&vJ5{_P=vJMpdkck&oE2kc3YYzQ>zul_70R7Y$an~aRE<$Bij+g zcBc)njYeTNdJfDTyRsvY#)#(F;Aiwy0}_$5msRz3M=*tKbI{i3`*krrf$U-i(Xi%} zD9P$?>T}vCA{j=vxz<%FX|}P4n+^E{(d8K=PVWWzTOKn7=!tzC1$>E(TE&Vm2%}Uo z(AHE@d3q&cD$^3>6{5cZs6a*T6O6?C(k_!Cjn(9;c3_L{P`#wzg`}lJ>;=1pV$88z z>B>QNW%C-a1d*cGCkMTG#0)MLX-BK7f5E(CcR_379b50Ip=$xWj7sC+(bc-a5Fo7fROZZn|KP8iojOk(# zCf8O(4nTbON8Xxlx)!bfUIeM+-jw$5s@Gu}+3TUNXDytsHpvHwcDsBFbxNt4dEagN z{gz+1>-WN*0(~%{qH;vu=vqq55L~jT0rGAA-Kk zk7`^k%xXI7OQYyyZEVaE2gB z%u)|dnEP#~D7<#aq?nM`4s5fe#w>`KEbKbsA*}P(`sxUHU-0HAo;RSh=sJ598YNm^ z#S(<$k`ad%eo$ZdW&<@_HT)R9O5*c@RgW3Hhbfa{)7nqPq}K3IeeYS)cT7=2a zwnNWL**WNql_tEhixA|--~F0T9A?8kH)TCImJjL5nC9_9>WCVRry===c!R!lU*(>p zT`vWL%ZP+cMaENSZ8^YU#3@!;!g5etiWui-ghmMm#CP9Ag=C|qD{ zZz4R3jnKat_By)nzFy7V4wI=}Q`>SMzp(S7!1zh8Z#Q!rH;kh<(X$i3ets{_pyP z*Af5ul{gpAr%vt_^n^ouq(d-k! zwTKrG6A>SQSs>uXMXQ+@I|%xb;uw9{u}0@o1=Gxq3uLi(+(IzSK1_+)d{rDl;o!Pw z8CW}_a@;b*Y3i-vC~!eQ_gbtdqDkpcSya)k&d_GRa9ds**FgNLv1WIrmDtdz6kpai z*35lY2dWLC-ZY&Zo44Q;skum4aFIe}PJ3&$UY8%^Ney_ZsMzRKs9}DjD55#yimN|{ zmK5(+0ksi3*XUYpS*ThOMOxKutPArKwKvdBn@Wiz*vbGE&p%3eu|kL!sWvLiun9NS z7WZD$jMnrzdrs#lbT${?B+falD@yQ^Rp(nO#K~9b)9*rd%rco?)hbEKE*d0uXIvk{ z$vpN6=UV3K<(@8H`)(ca)qbbM9yW|GqXwf|1MOa7-q+sR2*g6RR}(aLh^KhlPkeYN zL%f*jm_$F?M}J^XmlP|_gas2^Eu>>^(Q?HyLl$z4I>i!uk;o4pf55fe!j6{0Nw1lQ zdES-pfeM(6d3tv^T;lyfFO)FP5j>adhm?g;HDDQvb+5Bk%W&JSbKqNkR43H6Opj`E z29jx!c86vDiZ34@S)vm;WzvE_mgq2d@h$!-rscVWgT*hh!9|iczXLtMfEh{qx0>q_ ze9W!=Q?&U}{-*?hF6*;-k@P`+)dYQ1&3Ns!lH}>5%v5dhQJgd*r*PD*Jobg}@ZEZ2 zSvw*kfC>QsR$+~~ZQizqv^brV>ai&n3WQ++>o*mIY2gl=HH7ZmeKtO3Ps(`?+-5Az z*N9i-E?ugs6qPA~o`D`^byMt+@ZQ4w{=TKCeqh|wyZ*~~jz(D@0rMnJy5wB1D2d^q z{kD(ZOn~69aE*2K#YS8I8hbl@5YGSFTiT86Fz>0IRnG>c2_=*=yh(;%%(^hj1zZnl zEYOx_33+j>H+y1qM3}3(7NHqNw?zietS8#@iRmALcO>!``9T{k)oZeYnfs2k?>AP= zWf#P}ppkQBQidTw{}shb4s3WO$md7++Y#$X%}vq0@Tuks%c0EMtBakbx2igHyEU>mq&*vXh>3D6|= zY+*$Ad})`!&ZOB~LV4ZCw)2>#umGdX8pnOirC6_7hiQiigE^XdBOf!X?FVa`L!E53 zKYki8$?U(O6niP7`8i`bS%;nkmn(u4DCqw_4Qy-!pI5NBi$5AXVFMD61O9^j_s;OA z=%=&{7zoJCzhC~McFEDGb6{{MczXa(A%K(nB{$+};h_>FW^*8NGPA|~vgIRh(fjrK3E9Ws{@|!M zZX(XeL3n-~ALN$3Bh^qWZ9 zFpB+<(3SF=Nl^b(7e}C#BMetZ3ICB$&xf9^Ic_+Ipb)Bi$EAv3PYV-l{mvHwf!Spl zYQ2Yuaq}^7@wPyENsav8ae-e$l24;Tg^wW5$UTp9fq#YWKf<324bCXWLBYC=wH!RC z8PXrK!U~FGaFfu`!zW;FsH$=b9O`3Nx~@0RNS5KX=bCS68nEKX6+v1~1l)ID;V9>(Is z-n_+$=v8S?8w~`%e_H?h-I%(jB7^|1L2ay=d5tyb`z((%xbg!gdF~4rh*j&!`u=+= zW*(t(@pK$9w1=spz+omZ->)E1hBB6<&~jCv|5Xwn98%kb$BId}v^x`wDHKhtgnlXl zGoL=9`27#$)Yma`UFz)94o_)SI#V*+08>0G-4P;w%|2OGa}$fna;I2QqCELlQMM3k zTifr=p~fWnM*st@Kdl|PSGx;^9c}DNpcfY2#L;uOo19N9sJT7 zdV<=JuQJ+4Gv5cj682kQCE*o>GvN)fp;KBf#ZhB0l2s7&zr)Sqkt`np8U(}>6$FI; zf5MFvIIVf%fw_$NH~Z+%%(Q7A3R$wE5lwJ#CN?ai7aO~1(wJP`5EQ98?lmAGATynd zl@nBlXlX@4Cs{ys?j8+JyF4(#aOhpTvZ=Y!{cZSfU-KuvE}h#=dVB5BfZXd*s>{q} zqrpk*i@?X$#~J6pj(i^CAxp|WD6SG8l~HLFP}%ZWbbVMPD-;5m2e~ud?dO0YL`X0{ z3zkW36daisJ4S#ODE#2mV_X!^+UJ?1F2ht=9)<*Ow_|&HyN^LgOhbnm_Hn2UrCAC0@$<#YTu!48KfLH8;jP z@RCJt`&!mIrjQqmo_Uhc61T2dRM{b%GCD=x#!`|Moq2HSX2n*bHT7ts78~XR3&OrV zq>NEy3-gYy{#Td3PwA5w-XQNhK;%KOui?#jFo&tNL|uX ze?SNpi1PaKB6t$v((3J{)!@L5T+P>XR~HzXtQ)#G_w(0_HZ^a}u92f*W$7)O3lG)W zFQSKAiGcR?^Xoc0JHb>ghcBsqV6|pt9wH5birLEs!)! zii@VMgKA?tdN7{Xgd;Yw+bPw18{q3%dPiP062 zh$c_{N-Iyx6(jyeS$3A^Jf?}B^)ecXWy+bd~>DmI) zsk^c#V8exfU;#u+`q&c^u6EbaQ)lvy>P%0CdTkp!j2iq7QT@}SQK z^C;2QJwk=eZqo`n0xIUGI4djHy&#R5T)( z9#YzBPKA%==gB*=R|}ct$-;(!YMDAhBfN5O^x|wP=rajb+-aG@0}|5aYPlebn{!Bb5u{_Q7EbfUKLqhb1!I~XBULb{qjUi@XY1|g3AnRHAQmH zlkeQiD_#M0X0!63J@(}9?O3ZtsI;2mR`l9vZ{15z_!VWIi=4JRaURz1Ap(L1)7)T} zzoPDuXO4N9Gh-PU^0cFbiv2jMUL(K;Gx_!N(c<+51DZK9(y7rjc{9W1(#{UP2*l!l z#_zR;Ffu-)wGbEh^{)|%+r(5h6UK2Sn+04Vzo`TNocx>!8N4YN+0{aM!2FCOv^Z>A z*SSQytD55OHJO>b=d`j4vnVK3iQ+5DcEAA8ifJ(f4w5}Hwz3)-5F$I{T(#B!xcdP zJ9+{ot&5Op@q`6U6izdeVGV~^u&wIo@ckN=9pdfCbD8SMJeN1||ljm#t z=yN<+`B4{So3!Vza5Fo9*^eSls>?S2O_RZ{5&AldjE#TxL8xwkoA2tu4E7Bi9=;K| z_xSJXY5Q>2S^5vw9-%-$`2GWHj9CESuEL@s#%Qkx{`LwcE=`zVI=Y!)9HQt?Mf71J zW_h-L!$1Y>m8p#Dlr8tNyJ2jDM}s>dG(b4=H|V$0h|4C+s3wbMitEW%&I7mL=J(sj z1?Mjj`PeC-*HYm9&RnCmbUJh;^N0!ra#^FRNL_uDHqxWSU+yYwKbQG zZ|FL_m8s8o{?=I~P;~DRijAuQckTiD4ee#gD|}M+lW&M2RH^LAyZM<2K`~sv)=mRY z!LDjQs3od+j0IKC`+RRaCc8b-f zFXdf3pIB5Pln@e@fs&1o_?*@R^HmCGiySX}M_JHWBtAx9N=QBn1+|9c`ixY1YRkqn zYL&1j12=i<0cAq$N*!zs{B4Y0??9g;{?aed;?#sCF{qXGz}-Ynd_p~3iom4#8eoYI zK3q=GbOUHp5MV&ZX>kl9mY3p@LQP;a=7ZRUMVF;vI>xyP_P3v8UJsaYThk!2fCvF)K+G6OM!Ek*S12_2+~>}l5O0Uc*TQK#;Lw#&bC6*$ zUv!gRm=Qo{%$CIxmG&N(%nbJQMRSODrUfZU`ThUn!G3qt*hJ7=w8WQv`a2M(cm(0>1YJ4;Z*`4ZK7u&ZrT2Br$%VKS5#RD^so=Y+22!ZR>C zLN=B7$#k=kz{e~U-k|!(DfZyK!!FafxnFOUY)jZMC4EQsv1XhCsKY#twj&T?M5W_9 zZ1CV4wE5&aj-H}tFu`vDj7C#9@D+Q1`TmXI=*uwhMjdc&-P)Zvh6+9`UPi2 zUP@l^Cxbj%b@hz8yo@xUC}uHjaUK|}!eei&IouVaqp4V5W&%5j;1ZBt)paXdZw|Xk z+f3u1W>lA|Q?YHH#z%PK-(42qT}sVu`TR#=O(i58SY92`iD1(2l9O&V>$K#I+>wQZ z=X;JiLNCqAY*80&MRctp-ker*4e5 ziw8PMX9%S7_(_W3Bt0OKB|0CBqlXBKuz`fInqtDDmatnz-2~!Z(h83ZF%0B>Hs&>A zSeVRdV!@^iqP!_%6tXtuKyUDtqms$VHjKH89@(GBZhq;Z!e@ITz|h zo;v*ZF8gekiJl1#0wVQKI>h-u5+e&|BU9VNZhzFoyaNC*S#8^SQ5@rYC52RsYM3ir zWP#{Mu(q^u7KF5ARtfB%*=i?CIS~<^wCHN)f{3Gd(~|B}{97=;z#Yd${8ANj9^ID! z93FbWj5Lj$3F;!Q&+PU3&F6J%Wq8lB3obElW+s7;Ga!&14x|ZSL5ss<_+;Vo1K)WPKoe zNRz!Svm|D#84E$RR*cro^>x|~eUC{Zg@#M!*3Ll=`AWSq+_jE?hQ)^-Y{5+JlCGfz%-4}_7<*3wE0?lfJVcokLW&OI@`S*)iY(0LBa3+65 zQ;P(&pTcWb$G2a#$N4)~HmhEHp7^ZQx`IOtN)5q^)+1tFLZQ@<;(L#x1864t=JO#X|w>hW- zc3pjG>j*Qx!gFzc%{k(EAlsR!w)agl(R~ANe4cY_M4Jq8WDX$p()+VW^_!-DzzJjj zO|!o|)EqZDKj`l<8q3XGqY+(XBQV4ZShQlsNc4R!H-3-HM{E0gYbUGVswa7yhEfQO zfb{N-4KhmTK}S2+ftZ`F|{T(t1Kxt`IB zjQx}&;uo1&lplpy6Bppb#>1%!hoyk!q4lFl3OrTBjLs&r%oE=~1HPmbh7Dc=x?ya1 zh>iiPWabvP{+cJG7?vpmR6U_cLB5!V#wq4#e=LCr3-C*3RF68Pz4byO?m-VFm;3 ziaDdy35!X8A+BnNO{>Rz#5CNF1dL@spgu?IfQGeBtqzoad zgc{i-GpRhJfD*+{Ar66#5%ibgs)WwoBXfKBwD1+;`?u(i|1XLYfd z$x3g(%Iy^df~O5PvUwX>l9jNTDx#Tl|F%NpG^$9O&nqAk6$@L(xDH?A#za=-GYxa< z?T^y|?)=GZKXYs~s`=e>#qkF>L7%pXKQ#$*vw2V~WL(%nx3siv2Bp^9Ei&I>-MumN!Mq z#kTRBJ#(u%63wBVNYnq@RDeCsN^>8c*OL1acme02oK5@NxK-R|RapPyF{G6YfEkOU zoWI(mewViY^!r^P*FbDW{Edb5-}h^Ue2U%UczafxLkjh+7oDC;s7oM$3-is-hte@Krn*g;1Q@;2T|WRe%XNBUN;Q|9;0qC=0&7ix zs>5g^Z^~V`4S(-6I9sc=Jty2S*!{l|&Z@D8UYma$`Q!N34bq47Gi1{H7g5gY&`(yQ zZ`*?cYRW9|&eps{nZyNq5x!h8@hGPgnzC@1vtiHhLHJc0IL0F8*DVN0%w?}g?USDS zpH0v6WFx`p!nsu9&8bZ_@kEY@Y)XxQIIAG#;ViP&1#!YWcBC}ulK7fT{y%bL2!6aF z7WU@Z6OfQkvAy8}pyzr_j!p|smWCOQN6AWy`iwh{@>`uq2{}(Ih0{H27l_c&Z%9ZC z7pTF|M?fF953e^QzbNC-?24*RKi!kMq}>j|*3)PJ!t1<=HDj)Y0B1fF%)@!%pQ>B& z;HcV!=wW6mZIR81scDZRUW&Jqb{D=ifsjDa?YUFWkw;Z!cTg*4s zXFz7B8?$RD2@yji4>=0C+SsQEN?^QXDZ_LC?i%A%tk*m>xiwk5%?{^?r0-{}qRiCX z+TnU6SbG{mBcx{^&OL8=pKjVW9smIF2EjSAhN^n2_p~uEV3Cd&?6VFTat4>YDeI25 ziFFbK+o{tzu!41;TTjUo2->6(WoNcM?o{%@XE3c+BV*1UC5pwY`l{7r>mHyCt*Mg0 z;Jv;Fk%9VDL9mIE^L)!LZ;}|^MSZKTMh{E&f&b5axeg=12T6xUX8i3!EM%)U5BL4T zeG$`0=DhBXlB0UgeJc0B&ULm&s`#;E^&Wr4L?gbOI;KuVk}OJw{ulk zUqwg2F_gPyuI4tUaRaoldlm~*2m(;#-D3yA=C4;k4Mya!F;2HUeaJt#FoYi3sB`AN z(4*~XiQE%UkeIgz4-V96a>bsqW&{*9%Q#Czcsu;9$B`q#d`6O#z%RBpafCj^s01_R zoSFSuMJi-l?AiP>hrDu!e45OMUhzunXN>nW?+^$Fi`gQdh~SNHkSEB_p9Mh4Q`5`{ zDy2i%ciUg3Ok>dT$0BL_R!474(!LOrzH}r_Xq;Cl2|IFI5t8UQJ~4mj+qCp4`1u&8R<=I9@|+w3VIzz zQ|P~fe=3bUwAeNX1+rPqO5{Fdon*CseVv>zfKu4Op)xUhfaL-BiRD+_;ngmk^-_1hFO-@5rpgOZ6qk{7`6I)fmxG zDIac%UDZahjZW_;)j1VOEsBQbJhB6BxU&uUu$;?y$=sUD?>X|BungWABxrbeV!M8# zO*~zvzmgFcjKwcTCQ(XUjldFL{7#KaQY5W*hLX^PJ;KQR4V9nvy6T!cWyMl@b>L(U zY||29hzkwq!uT-X;gh+cj#G(ib+$B0ty3|Wdg!W--FW?sQv4ud>%WG?w%+S`cr34- zHq`dc;G;8zRsV8ZTXSHiXM0!^nI8}^a!6nIYyxg_pXas*g*n!JZon=N59-;Z5J1i{ zVa@NU0=tvN5X^rAs}UZ}`sZjt_OT>@NIJ%uJ71uO_`FcI{^qU;6Jzzc3Lv%`poz(O zIOxf16l8WaENpI#N$*Gx7l;{8Y3B3+WNh^w&>ljvnj>Fno&fwTO1?Jb)hLr52b#?+ zWpP+DAr*lPj2=T9*^vxt28A;zgEOf^=A0q3WSrjcOlGo zKtL$|=M^X*gNdz?i%Yham7&gZ^1s3O+4wADCORdmDXLc3l_zXK+K^T@B-4<5A924H zOvINhtvVU$I9W(t@XvK>nS43TOe|%pCc1tpM_4AabY(cRJe7eav$S*;nMqlr1yJFX zw`;!yx^*~?uY8Z2&X+8gS^i0nSwHW?wFhVr+0-wfm*rsT`CYpAXKlDQ0n4ohx;+8Q z9q@JQ=oTBQhs}|T2>LD=7F|O2XumALgaYc&v*#xnkq?;!zcZp!`Wl0Mt*0yk_8>}a zxKhFirb_ooWoR)KRQhRBzdmk(cu2)46T9+W^8S*TxM%TPTs#HEA&N$dQb-hsQm}~r zU41ezap}->m|2{Ega-65Y3uHMU&GtFN9F#mK~@o&*|Z>BI^A^46rJ^0Rip$*+0?iN zBeN_4O^j_hP8t2d3X+;tsoI>Tcq!qY6onpTcm z#k^vNPNd?#Uf0~mEQZ)c%_B$Ktw#fkYPk~F-Jsab$YNR_zMMoaBf6H@cnaCvxa{0O zY~gcE(Ap-ed`tHp5Qa|}i!k}BCvO?xC~L` zglFr_w-U0D5ux=rY3AU}(fv<0bBT+?fHu1^yc*9P*gq?9A!fMxlvGuSBgVQL>D4hU zr_|TQ9o+U3+4E_{*zn?~@r{A=nq566;7=PEFq}$#!kye`%|6li4FYDr&Zh&)vg@zO z13#kzK(J+7Qw(1lj%lq^maA(kYxEj`{$5WJm@*7y=&vdE2)bBhr(w_{m`a_)JIeKZ zD5dKA>#C+sa(+((#WE|DT&V_iM^$*<#eMs*kOTCXF|KIH+Cz53;QOx=Kg$N6C!qS{ z!W!3qXG;M>imKJ|Eh%;M{p!dG!aPK+^2yU1UEUcQ+0g9DEyJNQgx$qVs5Ik&b<+!m zYwjGpK7?jS@WI0I)6M<#?kWRYvMwFoZG^M8oE9Mp%JZ1Z4v|6ieIapjl%9Ve2K!(6 zgzS~>qyub}ipJI7U5YzmpQLPYy)u9T6o^o(phjoII`1Wm<+_J>Xwy!EE zxf(6ycE7M*Qu5X_7LF8!{Y_?pw5^F^QGL1%#q;tZ`h|w(N_c9BU)2~yYtAX^Tt-^` zxL@|p%DH^`7_k6*WBWN~c7#Al7UefW44jz{E3!zklyO9=1Jw1VlEF=%i}c~w^=D)2 z5z)Yum)^uF{-~)(KMS9Qzw1$%u__p7hK9NQlmWX9y` zfa~#SDKS3?vq)7Hck<7Q3na9o>2nF~7=? z*0|9E!i-|ZM7u~6?BA8Lq*3Q2&-$$skCuCXY)+|Sj6U}8a zDbjS81mQ3q*k;yQszY5$4*2|jWcksA6@_U@2YVo`X9aCm#kz{(Yl%~f60S=qwM;QWo1=p@fgRD%XBsrWQVJjs> z71wW^5}Q#}w^UD+WeU(5$ru1Bx)hn=RNdJY<5Mef`+Am79or_(yos)I##@QRD6Ku?Q^9wEgtGI-iTf2sU2^UA+VvqDP_Hp>IvEKS} zm0G^g9`%M&QOVV`j7D3fpc)M*I%v`21u0uaNrcOs^nv*l$;<DO;7IPtQ8Jnk`42_xvl0T87HQBeSfZ9VFJ9kCl5( zpz1&QIg!nP8-Vlr#e|Qe?;{HIHC+3~OtgT}DuTS*ofm)%VH`ATi!vY@iU*U0l}!2z z>l?PW@rbxx4n&49vX@DGu9p(eH@wX(`(pfCDvE8M(8znh#AVKdwk65tow&rs#LBg9lg9nSw zOBJ%k#@W`Hi?jz&)oj{o+j1mvkK@6n1wD8ewVJkDi$nKTFa0NeU1>Rv65}heihm)3 zzQA1O$52O;2|8y|MY@(KxA?O{Izi?EvNJHh3}AWw$a$xVyeW4q9hplN`n{A;s=A;? z4;Ks^jBgX1TS{7M8I&1>NN9QF(jj zi@F7=3K`7qva6wHek{iEa5XB*k;ouRHn=}P7|(1SMO9EK9bvxE@9=lnQl6X^9H0Z2 z{d&+^iE&n z!B5$=RR-c1m=?4Nq4sdD_KKW0mV92YbaWHS{x#~ka12*_`Ad;M&S24&9+~HMd;Ee* z{Y)zFO>%hF(NJ|XRTG;{35x=G?BIcwjkfy^?GzSYrYv0bX)F!jt}J#SFVFh9bWw(F z<2wZmw+!~>;_ZzTA1ktmLOh6&@Qv+%GHbGPxe|>}%57>W^H`HkN@v))f;!o%xH--$e^@!da3y`Imwk+Cb0e)m9Wr4dqa z#&xVCPf8k`@P;?68uPB;AW{gnG*>FXu{PRz{Sv=zTF=IdhklV=fl6ZDqS>)@ul?@G zx9ZaMFMjwZSgt__%xFaBM0x`|LO(!{9vSgX8p!Px=-M+y0QDwc}@Wf7$jhn(_&MK~nB~Xzu?u@`g4xwOen8SI-^&=pe6H7r_CxYAEZmJgXSS`RH%swr(IjH zm==hoVHCTe?@&&N%y)sq&mW)Lv3jae!Y;dk_-`18=i>SzoKqnQ{A26?4lC(v`Y@NZ z;0NJLOPeuT2OBull5l9^0jSov}w$y?k>RqR9Kz(V`Y1js<`(bM1OK24_xG@KXC4 zO(CD4=%tP8-N{`|8km_!u%VMoCym{AT-=+!+-bO* z2)M<&x}iUGGPetMW5k#1aa7yC)LWV9aaBK31BhvIa)hXJpKziv)574DqLaC>D_`AF z!7L3Ms>CT*_N=SStgB4=45B3J2z?x3^(lsH>p^=(NCkkR=mMe9hn6WqmhnoK$@MT1 zMEJ!x#N6naMTACLL1DH-#S1ZIy5VVCQEhw?NM3e3Gz=WJK2W4o@(78FCBGhnzKElg} z8wvP}#_$2Hn}ue*+wJuWG@^P(O^R5{f(9>+X9NZh2_Evab4Fv=T`Fy*&53-Fm1$Rw z?k)JjbD20PCnR2}bBw?Fvt>Qxo)COyolTKK2nSWcy`Wm~*IXc25fEN-na!3^G3q@q z*nRO%`Glj9Om>XK4^z3wPnh^FOU&BARcGsWhjoGOdzMfV7|J;0H79X8_o|E-9YGVj z=^?`6#`wZflQhEbVgF|G*V81%_|HbYq{3TUo&ELAa1PJL-d9T<4dBT&Thbo+opa5R z<+`Ui`p49Ae~!+$P5DWoD{EX>um37`5V1nInIXX#scL}*(O|M9yYEkJj|wuJg&)H{ zqFDkB7?m6eu2`e($nS1^y&WHSOUmnLOB zL803Yi;%P9vAGL;xln5k5yWBxIKUZkJ<}EPxqJGGNUdFGjiQJbclS?Xq?`?oor8_I z*P_!ARbGs-oLVD1`8oq`xh)v~tRxrz@)898naA2rQ`n?0b56E$QAPbkgYr^&Lc_Y_ zGOkEY`;AdZoTOM!MQzIbn~svwW^};E9RiwQIf4N^qv2ZX0Kp zRd3DX{k2fQp@03u5lJH-j$QiFW^A>=V}Ob)6`a8!XLc+0!5Q<5Ew?hS&anJp;#3h> zDp$SHyRcM9B4WcM|Adx-_uC%Q8S_I?EL3yXIDfS1c?x# zQlA6?l{tc(#WSPD5ky#?(Vt86KbPG=+4j&hc4(YXBBpEq*`3j-I)$<`qk>E3E)oiT z@|qq7fluL)Jw&Zv1{2C%)RVWg#_Ku_0EbCGPl=uW&U?b0#FsmKz(uPG(=p=+E!Br4x5i4zn=7 z<4t0suvJ+6Wg;tVJ5hsT8@-(yuKd7R-39=r83C_d5azqE;K?0-11NX`+lQ70xzb_0 zF-+KckY@SiN!xwu!LqQk`=*Y7LjTTRb$&9LQ$4aTQ(lR-yN6YWUbSy)PSR)$lj{+xgw0DE7j^XHT1-@Jr zYq`@Zkq)hNB7hVI^ePkTHhZ5xw` z&53O%GqEvoa`W3~*S_bRx^=6%s;jH<*W0VU@3YotJ-^?8?EV&I^{}21!UgQ1+gFe> z2zU^fE2&Mp`w)@;g*fD=@r+$^{P9~|%a=`8f_b9Xn)@3FH&DFlBhj0yY-(iiW#sx2 zx)X8EV+`ec=~5~|dYaBLjcd$i$Ye=K?Xj9Y{LO_Z_utcDa(IY9(dV|)2crRfwnj-6 z$NA*!7N2RsU~{vYREDMt*%@%3dELXkRlrGwlLlAyz7rGIMur`(l#qOXOj&4e&bv)I zN%O9tHrDyqJ+0OsFrW|0cWnMI>qK!K=3ZTOjDQ3yvp?-&?T|+FH7Vaq9Ex8oGZ0tS z$@Hz}IoZj@J${PRZ46H`hvhp;*~Ix>$ym#u3}d`t4MH*r#lA2202XRbzzY?O8^Fd& zMu~90CwP??k<=bkswy4F<`*Ao*iV`d!L{juS0@wc^{EarRESS5z`BE%*=LYL7mPH;CtH3BLaHU1j-Sq zWzwE6;+)&2$x5tZ5GibO#2IC@& z3I|IQsX#FbQhC134eZ#rX%ZffK0F>1_ZK~b#;UpyN*Pj$R{d1*%tI_wO(hA(4kn4# zHEp8Z_YDUsE;2mi6GyUv^*DXhu+v#m0CA3byC5@GT7-Agj+BS zb>KBIk)(R!@SUEdm#M+S`#4l4G9_=@;vS*RghU^9dl36LRZ|7t?mZJ48FbeUxO{ZC#Y1!i_8wcVD%(-$b=X*yj=xBXc+zkR z?BY#{GkB~UyMN6Yt$R?1U&Tkl%mNH)UKk#7$}MtnT-RUysGkdfHzwDgvpm9_dF~*Y z%`_(eQV`)*nHv#uEU8S@CI=B_QWN4H+>|?kL$9nW>c!XRe@-Wy#T?nL&mMOU#T#>9 zA@%!T=iegelp~f&o4~le|DuFM>1|OuA{<)fN!!NEgM)s~1U}_TS`}%vKmb({u(hl! zBBc>h%h=KgML!%kcUz|%VI}@C`vw%fFwF~+z2hMG%XS~ff#0*Z34a|3Y>(T;yLtLDHCL)M%WIzZApNFwG~v3{c6Ok>ETZ@d?sJ^@KRJF2inQ&kzPf`+q^qobL#sFjPWvz76u z5INYZIRmyJ2m4Q%_!OM~p2e%!eHxTV{>_ZKI&qleQ6wZK`ON|RgN(yCs6m<1<2`at7uM+s)}O3Cg8qJQkcRLI&cBVe zM3}?CF`_k_9Ji0Ya~EIFgeQhHhLDFA0Efy_Z$ZRbG|Sk^xWXz-(M!cBIynQO;aj(% z3c0@YSFHwz32g4u8FMddyQticB3CJm%<0$Jk8=8WgoO8}WfPJk~ zKJgbh_~wsWl8pM*H=;bxBmvv!{*87mQkT1?r!!5@9 z$TUE}R)sBBIhO0q5454^>8#*==_v|-kM^C=_0j?(vHNjMJIuHxW#|iH(tL7q^S9+uP*_!Y{eT zg1s#EHGA#>ysf7u>m&Sxa)BupYnBhX`AgP$P@Z94mEg6*Yvr8t*Fviv&hgCi;kTmZ zlyfI6xs?2zb0L7x;~171BC8f!BvHBSUVB=BBto;|+oO^xRR;{SIk=+kpKEYF=MTvl*0lBi7?Y)=@ib^YNTI>s ze63N{NFbW0P@NSUSMC#}8as(jHQSVOpIZryzjqM(R>J_cQ`#ry&en+ujhn$!-&tzn zx{wgB6!ZOdhapaYd$XVB6P3#zBYrDBv zz5;aku#Rb`HMdIv6nyWL3~_lj5n;DRt`Yj8w3cbS+eF=}(_b%BXjc2owszr!I>cor z%cn;rM3<$W2k2GQ>i%S)(}RCqvg%#GN45q{@F85`(4!t_H~16tDoz1r~@h*8V?y#WYE{& zY;rPIee$|`-p+qMj0%3dy<_!Kzmky#6h|kuQWHvYMR9#)dRtYo6Fq$mE`KWGcaam9D5Ct2sTX zluuL~o2Yzlw6@yoAi3i`#Q~neOJr&De(PAvA4d43YSTH>I`y&PuNTCLsj<$*T61q? z8bht@r5G`_C$<;amqY8Z-$XQ-Sc&_OC0Uy9C6?Y&W5!wkrS4Cl7=~-abe^I^N(H;m z!d~M$53)vnuD0Uxop+j-&#;b7Vmdo#YE5cwNUl_hn1m2HkHkV)9t7OQNSbCCoh>#e znC}*_G=1d`wJ3Lwf8S{GJLwG3c;&>{mr0oa?hbAp+tO2wLTB}IXC04M}ydT7LiU1A^4?A;mt_^K)os2Xs&KKu=IN!QTYvV|m+>P0H5-grYAsy$NZ)Ni=4 zL5Z4>^~AfmL2|a#k63v~9j?tIT}5pALp&FOXg{bP(eh@w(GP*ZayzPuplK(AeoUC| zK@F5{^(%PDDo2`El~d$5lDig|?g21}?ftLpi?II|c76mAUXc!5!P@8BuYt5wNjTr?^SkEb$E+O8rvljS!d`q_>L2}GPh`)I8}CoPZiJ3lNxj?FLWyX zUSPY6K^_NfNn1c8zJ9sGlVx?o=|AnLkBT>UkcrmA!q8r*W({73OCn zW~1S6863EnQ}(a3QB{yefdx!3wlopTIOhxu}J z3rCKt@z2)wlTB=E;b?v*wIQ5RGQDleV**S-fQk9p62GtxoFAJXjC9?wXHEuIzx3B4 zdMLAO)gCrcBo;rmIFqz`Q^q~05XeW{79LM&=$Y+;HK6$O@DV69jHz&$?}j^<*8ycN zxsZAtlL8Q@&C0`+M913UrQcod7_Jx$7t{;7wnopG65&DN6Q52+GrF?r4L6J~Inv(p z#m;#wPu72My0Svm`(vI#nQF(O4x(Cu7~hPKM$kwkm8YmXMFkllEiDoHiDyU(xDj2+ ziLWt6(z8sV_Xj0my=eB(y>Aopq-1BNi3GhOcL6qi@=ZI}T{wNxME$y%{o^g7O|3-m zzIU-a(L1?zENn06$gI--r)jd=+1m+*{)`00X^z-3@o}zeq~N6cQ)XoWh4tan;JXYYI^{SYjIv*C(vY1-`M}K*1_mmb;U;dv%s2ljL`@i? z;bposYgIWpxB6q$?S6!dS4gSjG$%Sd-)q|sxK?)d&m_s!uE zNQ;3RhId<;hbQZW1wD&hsJ_9Eo$&?xW9fb%(sQ z_e`o^dKTRU@?oW-Rw}qq$;0q2N2nZUTD2sL5I_H$^624qk?iGum%f|IM~_4L0@;;# zsU@F+U&MBCZNW3;prHVjY0_lkj7SQRB-8yRM0zkUc|CNSPWBkl+^ zy=^fA?;aVFn;N5bWRBT|Oej?YAdpRIv^AlWN(+KRpb_tq-m&`YJMB?X(neDNA%vMs z@1KDWs3La|=8!tMZvp@A7!}_`a|zDiE`&?2S6xIcS&Y=KSX3@O-y}59HD|z zNVM)jr0U3`nrP5Vu>=?+P%_5ZZpN8zQtnoBGL~}gtgX?>rkue_jgWt=Wpi=Me!-9< zpXS?ncjF`*a8ezbczYXKBGNMhn_KD1NhdD4B<6j^&!o@bhsEuPIhG^h`UUI(i>}Hg zblpPh(a*djHk&5|g+O=mj!JsyY`oJ~qr_PmsdEtH=?EEaFItBrN@%f0XekH6wSvNPzXi2oh(g}tAilz28OxjYIn@I3_GR4x0o|MMuW=F@p zDx87MBiolneuUR~StW=OmS;EACEaU6GCQCM3?j4sTT0dx^8XYUw*L_qnq{m{VB*CARRZb#pRMyhGr4G7NfjK)DA?wt6@(H*6#YKxUvX?4+}%!QAb2gU1L0|F!9Hpz1F%_1aSI0~woH3CmP@tD(y|*NE#}=O z)H&L{Ph zDOS2(Ru)z8JE|W%)WFFn_PgpPAl?wWlCM$toIr+)}=Gn zpB$aLl&Q!hb)VvZ>!2auB*6qa@3?iSA$l77d5yJ4`{!BW9hZ8&beNfk%3*5_+@+&$ zU)%+dTjm+cKBEQWlq*8_k(?a#X6Ymh=zq3Kf2FWsihP7(2j z4!%QpnAMrwb8?H4gb`Lri{yEdo101~owYR$B@iCBCBDPA!hy6rBZ0fZm7n&zA;MK9#?(Bx#z=lZQnQafV}nV1 z-$RC}L!8%m{`kM{ndiWWW_dol<=E(w@{uWlNo{Cf)x||Y%8Uu0j80oHQOK_n5O7*4 zG%`WRC_&I)3$<8c!puw1wiKJ;~D-| zQ^^N(G1LDP(HRr_jIC0kf?Zr|Q}txrYG+%QK(K971NR$5Q85H-!-w8Sw7$8+jn08n<9}U3CaROoQ`qF@*laoh?R{AOh2KBj>=UM=inWSvwV!Mk+vEfeeaZuu ztu|_Jy>l2-mBPYu59F;lejnqqU(B_xgKXMHKsKI9VY%q`(=aU-UO(B=Bu1b6}&RTWN8J0c6rs`%_lj-pRoJ+Kt z?F_GVm32+Wur=Stvf;~<< zCHA{mekmN%y7}jbZ!bRH6Ld$`Ej{6I@5AF5R9)k``6o>+A9orYp1O4l+uob+K4au; zSKh9>`gcrmp1dODILti-*mi0G`yST5ut~Gn2s*RP@z_yX%%!YLS_9@JXa<<8s0d;c z1(UPCB^M%x+JhQeAw(z`OaEQyWQE6R?|M4UL`yJfZEgQ-u$W)U#n)N$0n5qq7Fu!( zXvqwflILmzNSYDfn5zSr2+&Sb*Ttm+sS$@Uce+?ep9TbO>SPv^f>;Ptq|CRNe1m`0rS)h{Mt zty_t`Oym&D=;C3sfpjJ;6^EyaX$5e8l){>`V3^v!Y)dau7v^vPSkRZDPGIwNe)4Xq z%&f4JGG$)*P)xBG0~Q@`+>~j?37L>N7JR3Vm9J$3cVE8tOHesS|8(Fp%Rlfvw0QBJ zpQ5eq^Sr&ozIL-BvPZb+7jVlQzMOm9#OVpEcv(ObFjLgyFA9Fok>o;;Rp}(@e%k>Q z15}z4!@VYfIe8iY>f-Igfr|Xl7yQ)G5xm0Q8bS%XrCvLH%tYO@e1Ij)yJnf98tYrd z+*gi7T@%2eEGN+st$AX)+I4V$|E)`}oYCk93!d&Nt4d6fuT~-gyFVj~$g_|sxZiJ}fyVj>En=Re8ZnfPyvG~? zKH{&^RA{~_)7eTsT^brYnVYo*m%Z3PDO75>*6c6RR29(#XLpLwuo5A3TZf_9-9gPd za^vJdQYY30vk-~BA}vZSPzCkY7oV~dT^%xFy7Yqss$7#9R#ut7-#ZMn7pmEK1C=nU zDe9&wBm%iWI2d#)cj(_BL#`3(-HF2C7Hi+E;TD@$ftX84UNC=cE_5)Jw`L*>l6pow z&IfZDHHhGr%!ZvE)dl*=)2idKsAaz3W9Ctmrh7~kB&Z2^R z#mUjavE~>O-2|~Cbtd7%rfU3?(3KL5oeEy$G%APShSE4`$jcR+;jX&6ThCo2q29i*dijcgaTgm2Hfn0;gjzC7@OA5Sk^UVF;y0)X3Apy83znl`*O`2YBPBy? zw5L`h-J8zV`JI=iBuNey+;Q1~Emo)8_ZcMfyt&7(!Ew%W`35tL@VPrCkc5Zdk@*-6 z;1yYT)|P8oAHfIy^^|@D&5H(kL$HChAuYYI3EEys`48?$41Leroi3eHy*d>O672?L z!VvaMXgG9G`{E$U^1t#PQ2I8RSAwG#*y!^kU&Bn)1!LFxf5B%n5N&F(+9Q+I0t{|-O`D*F!DuXgXTecOK#S)Y_IZZ4Tk-t!ke!7M&| zcy`NI{@|WzL8iyBS5k_(#}P4a7!b{%x6SgTSa zJ9&|;0LA3A*g;vzLp0lTVW9RjF`qdVX8nuz5a^Y|UVN8;JM~?PaW^HO2(Z1neT6N3 z(^>xa*~PDWb{Q8Dj_>D}^;S9d=rSHH7$gz74L88YY=eYJ1e7L1C8=8dGfDa|Mu&eV z9@JoKWhG{Vf1e~CS7m}83Hp*Qm|%`&DKv)co?v+;T~^*^M-2B&-(`8l_@n{(k@@ur zX0}~;P6AX#qXqd9Y9K^q3cw%-xk^uQKDNAmUUO$%QFFXwXCe!j4Z*#;NqVtt?>T)v zhl0KSZ+@qz8aS^K?k)+B=|Mv)_lzlED z`>3Tz*c86M4C9cB1RcY#K0(*B*smDTPyflKreYDs1x&^hy1c{#r*F8;t^*hPye{lq z8RPO+YHdkbIWUwBZx+EH{b$M5p_}qwS+f&V(#OtsHOO&Q?uHtM zhnRu;J<-)SQr!Avy&n0B$yD3sviHEwS@-2zd*4Yc-Mb|$ z?SHtJgvy^t*=0HdT0h8dw}%>Ujk%)uV0rwCW`7gBwDD73qEle{oUDUUg){xe@V0kz z@0#{HI6aXG4k+28TvbiCo?=cZoDH@DQ~MiV21lt7r^LEsavoEaHp78dU61oeKX9&v zm(z69^6S1gAHF-!K{luGZtF(BhPuKwj~~V56Yj?}L5i~rz<`HLOE*d?uYZ6!=?j}> z?_rZOR3-6=W75!iAe~TRQWHFN>yk z-fOD#;=<*o3%Pebi`ea|nekR{i55`Ph&V?$y-N$MwZ4B|%1#UhXC>idanOQjl^2qx z!wz-4FS`o>oGy4KRgP{S-Duu?CD{g?CnruyO?0fPjhCYPUEFD;cxi7#-Q&)2V8l%_ z98ek!_PQr@r)1QY#*$el3Yw_Ld+8|uW~UC(zfjxsobN#Y5aPSva2H$r*;(%GQI3HL@eHJQ3$`xSOALl^pAIV&iAJ^+${&_?>bK5c9>_l4OY^^5@@VEtM^MT#j<0 z6;!c{Z@xrsW#1An&2GO1phXfPqQi%hFniheK?e3I$2w2BPSA5@jq!t#OZW;LaX;+wsyK50@tYXXtacb9GUzr>}*@$0Ek!6?u zX8~v16<|3ehbN7o&ECi85!orh3O7>8FqO;YUoSf!6);zDkT?z_AM;Q!iGJuTX?{sk8nJFtO#a3td~A1;$jk$6gz6Q6XYP*V;{ z-;yS{nU!<>@|G1*9`a80M4 z=P}Qp5A+m%d>QtnE;5L1^En1tA{sU2Z2U=F&D5@~5)_9RDR&IgeW;Qn6hG~woPAOi z$DEOvscINKF1lmB*BN&vhpftlQqK1#kuETH%^R@%!Irq~qHfNdxu2O|SW&I63{j(8 z4NSFg7Hg@M{?~)2x|s60&2h}=nQN&x8Mc9B&t5?IA`p76-8!+Du*|shqJ965QMwa1 z54mhy{H%%&kS(2&s2VEIJ-_eCPe3w2VEB(q4Q*}-loYp2(3inL?vj$|#D45bOkawz zCW)zpSfh0F(&xuaU!XAaCfz@ICj~sC3Aml)e@}LZtwe>vt0XHw`-Ny|_sz!ck9R~T z4*vYnBN;t;nmFPbX$%~V;-+8HN^*yyCF9QacD*^+&X1d6P_xmIw&#FdM_29E;SIW4 zuFCFXA=5bK>CQj*5}7Eu+~xD2mLRq-F!NvP14*dplrwP<-Zp3Iej}gGOIHJ7_zk(e zofIF7`Q|~L?7t+e)aBlkGWGUd@JqPay91Aww(bX(Ae!jq~DW)uU zu{ds_ljuo$^!h>I7S!8sUjS%n-+pO$xZ7vz`9f8tS_ENzhn13-9|FeCCv1U73W(RR zkc%)*mI+jr(o6eh?J7&WLPLxNJ&~us!TBvg2aYpQ1`*Wqi(60}S6>9Bwv*+&LD5?~ zKk04_NogyqoPvmQgO%82+t{S)IYn>Cp<$&elS;eIgvtGO87g^&;Q=tpa^zXp+J{au z`T?W5o7}jmATqrjy;%7~b=jG*q%cdjBpdS+3RhckOwX+I9FozKqz4cLVvGu0Mz4Ag zL@P1Qm+Tg!zsi|nX?;a$q(v9`Jl^n$Nm6o6s0o}<+lWS&RPKnR>#;--5c626etAXH z160u3Uu^_yU%j|u0)T-#h-Z@4Q=z}8cjQ>JU=IG+0t~8NCpBVT}j6{dr75I*}n*%|OU?H)%THm%$F}Vi|n@dc1CUVZ9F?bIO-R0&OS< zy3%E^9~#7Wb*1L*kXE-G*e#?UU>Gx?)~0_RUF5 zVr=N3E!bkoy=RW8`Ce|}?Sjj1CQ9IA50384WYGI}!C!ufbsT-a1JMMk7TuBFNf(S@ z5)Z@$HUO9v$bHwz#S;2SB4pkOB(pz}wR-9AP@k#MMIhb(VK}kAMf)Sbrd+e2i`t{} zC@7DTX{K1KR?Kq5~}&bU_wQwt^8R|hR~8Hb(H>7 zatg(!3F7PPD_Nz8n83F~M9CRhkPDk)N}20KVW#=3E>|;arp&bq zwm-wjG{S4AdC&x@ULrwP@UO%h)%=%)Y4-!N*EpjC0UlVSr*IbyE?8|r@gC*7dm5e9 zifuVt>lHns9xKP286z&?8yy*VPq_O9()G>DU)IDbiJ=apAc#SzZ-rC5nUTf$g9>lp zdPcwqLL#UXieB~pCTZuMI{o3RWMc7r@20|t(}x$+Y0yfdyb1j=+mcAnS0R{sA95nU z-qCWVCBX{t+?<@|LmH$v3@p`uf8t#<+K|oHQ;HQmay;myj2w!u=Gk(Km;Wtz6|uy& zF%eMR4TGiIIxn&=p-Qc7s*+*i(NRFc*mMCtiA|W`9fSpb67{!kjkou0ynYN*5_TG_ zmul`2LE$|1u4%nbT~IGJS61ugkg3FvV&Cr$zrDhj6I6XklgZa-3^GLf*9y7oLjO(j z%#E6k2@|cSXnX$$Bv00RKrSq+%W>QtrKckrNAqE=*ysoA8$_2rS!PcmB~}}=-6<}R zqF3U5Ue+_aOZ%EYl z5<h6X^v8 zc3+iRlI0R~rGd8!qa>M_I)%8Xg~!mlm2*FrklKKtkk!Di1$n4gUpBsh*&8Ss3f;DW zfqB~*=J)G`iB}=w6}8|=8GJjleCBTQ!FjXTiL*c!n=WwWi_;lnY_u*wY zY90yX_s^$N>Vf}k!hl`HPAWD*O$y=Y`roX~-69Vw&}SN?8urTEi?&pBQXeW0^@d}0j`aXc+`@0YbE)^?D6nK%TJw#NRB*hV68cKK`JxUz zJ+uaHJp;8|&7~|QNlJwGV2Z2+($$SDSAwf>o|uaIRTGqqs4Y87v+PQ97AB*_-Dj9e zg5W_K_cq=O^+nNyYW1;R%Au$5Sg}EJViU|L+|%_nP9wS~jm4Pwgi#4I;^u(N+7dcx zj*IJo%8Y<1@3H0DtV(J06puFtZO)Nt{lB!vMBf@tyH>kR7CbH)$XsZ(fq<9Vb*R}l}mVc0>Wkz%1i zx~Sxg_ETvQkGee%9mEhU5;-7wZ`ko=Yz&Ij@H8(~s|x-ecMrVh0gKO>Y!T2ouq`{hNMn=nu}!drG99D zh@Odf1$!d*NJ1*+IVI)FSi{Yge7%Ab?;)AljtX-#gLzSLUSneEIS!!it?51QEw%3iLoc0*!jR)kQA$<`jVp)`?_S@LD7F9UgnC z@{2|baDC%WSVq{+psa6oES@n|b^GvvoY$$n>z)!r9p5f@{Q?oM3L@jc1%@fV<*__` zW`>6_zZvY&qPKu|n;!!XS2evJtrdnZjkG4*^ck%5pw9Zij-#9 ztQC1JswZePj&{;O=3LVF7M`)g()%kNKi6Li8!vx%RYQV$3;Ow`ibXYz9+DCK;M{b_ zw$Xnnb-$>Fj=jCy4LSueXx;sp=LYbs4sl5CFRdN#CH~MQ&4?Hf)hNz{W)6$z&V8wQ zfuWtW21=aH>)6JQgDW%r^}-WXInSqxV16|=%;Vf4e>C5d3R~P{al`I@WCPsr*s|j+}p`yfi>UxhM zZO!_PXZRzKMG3p1b_a`IY>m;-B`Rf!%zLxaR$Uf?sI^Suo)Ej46lgM;)cwx8!a4+r zF>_OvCpgs)&4z?mj3+2(5L3qd18a{f7#2(X8>!?zUYCW_Mj;#!=kyv5{rW@dRT?j< z^^XN(5Yp&*FEZZ!t!GR82$O#=b7g_Z+-Au9zUa1P^ae`Sl*< z9@_H%c7+;D%jUg5v%Z+0`TzeF!TXQs!U>@RpK&2Ytl_+$ZnU)cJFmlx$_Zad4&c^0|6Sjxx$RBx za#q^rWvdCZTQB`#RN~q>AY5rd$eW7_@EfI{1>RXz%9|J2h0!1=_c`{rLuJ!T*s>JM z%sXJ;{0d+1IPFYfGjm*&a_Uw#N~h1kX3(mLOF=Lc{-zH3<&Gc*G*bv(g8;|lAB12u zp284?sHS|o9!PIGJ9V9z1cjRBxH2tdOz-NyBAS`MwkA?QmbG}0RF;strE8qw$hi`YFW5uV}){MZ~@tnyRA&BWKHo3pQT2C+;K zG+z&dO$3~)Ly-T;6H^*v4?Dpt0<6#$))ZNCvvIk;$_LmnH~QVL=xYu~Ym}D9e(muW zH#lxylaxDIchf}dugNj+yBKOQUh4hh@#ho%3ezzR@{ld$VS^q_ULvyQ-h`Tf6SBuk z_`C(`Ht*VX>?5yoS;Vb0pIg)Frm=e_*VbWTj6)kxx zqV?%!q10$HO6H$tX#F%H$~cx<_*SqX#q{pw86^iL}xjNr# zpNCDSi;NHMhseXNr#TbRCRN^XnIT=uIBzdD&x#z*Bj5Jm$G-q&=byb8uOs{95elzt z;y4OM{^?15``t->@8p)>`?5JYU2GZarnBlfEu-1zJeOC>$d_nq=Qk^VOB}B4ttxW1 zi#|)31tbn#+z`C27#yw~!#a8Y&i*x0+1k|49N=$kULGL!%A<6hPLt3i@Z*tijrFdG zeCbjh#r#c55!VWQI)aFvUWk|XR|1TPHz5Swzh%ZBjaQ#hav1Ng%tZcl5%}k&Sy#W0 z-yUQ7iV3_W)+LXq5%~AG8l3Oim|plJ8e~l`U*F$21@qb8U48GoBkYzsMD(r@d-RNP zNX^wLMo(J`>-41sR%DR7Teh2>$|cQinMN-7;GqK2HSQ%L9aI z2?|hPdC3uZH&?=pK&BumE}(w+(z!5z`)A)@>z+H~F(8a^;VFJ9jQ`aO5fB#dIVutG zmm~n`=S~gq5GFhf=;Q1#9`o&$_!$RC{CO+qYj^XX7>~O}z44rrNxK>%cyA2LySdSL ztnzwg69h$t*RW>6#{7pD~3 zO{oA=$`yDZ{Iyp+Sbrq9$II_<2r-Is(=4hnl$B3(S$3&*Cik!a>g+sI+>N+09C z+Ip=nPK>ilPLoABYa~;Cz1gk=N4AwAHp=C}VQ(K@%_Bq#Z<{{AKthvhdis~4%mE%< zCSY`N3Wqod@|l1ysMlIwwHn@^?2}ZyR?mqN8BoFu$o7$=LXoaEmz5^sw-m zb*HeD^dq!0DG^4Na43wMmrr_x zJpaYz2^LI$Yy5N$jyOoUQ87E8Q=_k?~_LsQ$3`GXnI`-}>zU=BfvJUL9DA3Hku1QvY@D;P9E|K!H|!4{9`2R8PRT>U(^u3V z6I{fs<=IZJ`bqT9g}R>$nD6e(*v>;;+@&<-6>4+^XEyDICu-H%N&3wcN72@f#++00 zg6@RC$CR2aGPCBzsh66|po*%T02hi~D?RfS+4RV%B}*WwZR7{z&>*Jth>DYIgba2> zbwm>D*&;ZHu$#3z9O6I9{S`s;i_A0rrKO`+)*Jxgy1l4D-X&P`#Bz^ZPZ2AtINRHX zX~~3w&Q3u--S186u(2J6xx|GC?#FMY$JYkqgGN$Pgbn;LP1&`J`x((XpbhbjSV*OX zr1rk=-drbY-~3RBE*Y!Z9(nP-Nd{LbKe5X}(IQX!eKm5nlT^8Y5-vYJNKr>o|?a z=t}-pw8yZ>8-^@bM>4A?K+l6)OF)w@_GI4TfJ6;ZHC$CW35? z+74ObsuQ0WnG_umcoY-)W)y{e3Olrqab&%Rdf{0}%M&lj9ai`YAjUa`r9gqf&Nd>G zmc6+A5@cp+Awx$)&3BA)HR@klc#UeU+*js}OG<=`dldH0vTGP**4=JtV8HY(jDv|9 z=jW#8p+-TNqOSy1P$Q%)uQcY@pPRFuGKv#97Df@$y!$g?sP~TnG@OQMQ~LF|th=9P z?nsulgH+o|lBykzj#Nta^V&cwkz!}u-ca~{bB6a3i@c!x&cZaiv% z^ou26IwJC0%b%XHfNzhx;C%FW^~w1lx)j8DrF{R{2kZ(K$@7Ftr9!p!;Jk9a^LkFA z4ebC|f?tUN2?N`!9?wwWy16>IXW~D5z^x!W4g|h;NOxR`1miTEcc8xPl;Zn550cc6 z=ZOWlAJa;W*o_t}O6hv<6MXxay8m!r3H{{=>PV~K3R(qG?k}N3L3;j<1*NLkuc7tN zpA)Ss9g-4#bd8Lyot7v3V2J|Wm<6;LVR8@bP!VE4#YmkpH9UfFfeqvm5Ojcsiut#E zDZc)+5Nwy>D($LDd43x=`r%n==Fx4^%+g-&yR^n}=E%&=qg*d$bGQWQ*Pa3|;mpIR z)F|f#n{w;9Vst)Zscxu~JIcbD_^3@ttZYL|R3j0)L=Nm9ClDzG7DsHMKd@sygAH`K z@Rl{e3F0tHsCqA};b!qav&vfZez0F8pe&Lfk!H==VYFfyYpQnIbFs~dEJ$NBr8WYf ztb)(2DOzi`dRZoxF6>M^3R#G^R+QLl-&QX-=2ITbbi9qEd<%MLt%p;>xkAW;3c>}k zuw|JMCu(mvS1JsOFT^=+vR>jp27=$ot%$3D{F){&4v*|Gdl2ocJV#TAc0~dG+yQIX zVY%#z&q#yV!uCUtlcA-Vc3Z6)mUaoGJ4aW?AWC>&I+EPz#dHHqa@DUP$hTB~3rm@% zl8LVMk_8wynuS}54wsr69m--a@`OZ5-g>&j?Cu?+SGTz1OQy+iLFBX3wBJ{k@Z{b= zl{5*Au8H5fr~^}A*s($SEnD-?gUEo_e8rXZHrxe-ohCg@4@A$u+CqEeob&mv}w6?406L4~K zkf1USrawt9TC~@C&!l_T%jxn{LNkt4qpdgV?XSfF=@jKyTiOhT_;FW)dI_$9xC`ZJDZ)U=ET2UWR2{IB{g*5S&; zMzU5tMzqTr9jiIopsb3sTEmG@pvbt|+DbXXJ=cWAsj>5MVS|U&yT?5BJU68?>Z71u z3wrFXDt@5yMCs*LXL?#YBG0wqDGAXT#aIR1JHw0~6KsAM;45cNppS1Oc0f~!QVzJ5@ zGdg@0he}Kw|20olslVjP;Y~`jZASZ5nZ2J^14n zDQar>U6m!Ci5}idgnA-fJJSm>aS(;urs&;{`Qa;9HuC+Ldu%dqTg^*wQMT>e%_Kel z(yXFaTidjV?vXso^b(bdfTuKM+E@hG zh7COq5gr-LQFq!Ms&QivQNEuw0?=bRL|4(0U0TxHneGenS-QC^Y-Q6{4 zu;A`GNN{Ixcb7nL3+}GL-2wy#NRZe!n_qS}?E5|co@eOkTj$)auBmEzy87NDQv6Ds z@=Lakfj4u-iWDfK_s3A-W@ybteb5hlIj?)r8%33;%x=wu@rvW6C6Fl`|3MK_5jEVB znCfQgnY&fX=Xo2b511FuvLu`7eF`#5dl4-p*K$EG;nF>WduvA1FmiLfhGKP3y$osi}qMq^#) zkLwvbv#-F8a)lU9jOUsi$qh5`jh7z)CT|)ph^7_9_m6F+9N=>T=V)3d@T6PC-Zj)N zxs^@Jv)9@c)^X$_S%3<0&@kg7EC~sknsU3Y0?zdyOxL+ajJ0&F8L7{C?`(pO*OB;n zZAo#ogfPwyOS(>-%sb&s3AbXe60HL=_P=7?jW69-RH{2I&9SB`@qF%0{G@jUn4O3A z!87|5<^exD_~6IZF&2Fv+GOUI9-maxGiNM5iwgXK?mIC-R@~hb6FFw6`-px-rYjUc zI!kak;*O(RU&!Er=%cUVPA^WUff5KwWreHqL@6R&YA}FRlkQShm41CBeua)wB3HR1 zdfGC*v+q-SO+^Kye96s7YoL za07ee(qr=n2me^TH%*vjo6sV~QQV|`H5%kDsU^droRcD}E1~C67h5roO2bp<8n&SuZnO*T zv_X4)RmOfV>E zhk%8obQY%JxE-SKF_1lU;DGQ5loO-QGkf~4<<4J&+zl(P~ z&7^92l}nmeAOc>fR8b~2%jVf_52ch2uDm*pyrUv&e-FtS>XtJRj8ws+C#>9}%#wzN z#R!6K#*7z)?J0H`t!~>oUTSXMuR;+*O%Qe0zXV3ycU^ZAdG*322e3JcuiHvjDA z`G73K<98`&rf6xPQmDH3>F%^#zH!|VR^=CL?5j{;Is&41Oi%A7`BE*x7r+kCy$%(n zsL{k<8mKxLLtUMn3tCV0jK@e$%}r ziEc5JyPwQHJ&C+jB93wkoADk5(gVtLtqG~!BzN{3J|w*P({eM9@dI4Ri&A~%TQ1AU zA3$R&t$;kfPueze9#UqD7-r>9P=g?d+Qy6r$o1c zMGpU{8Dah>Lm#Je2~H?h+XtgWsnS+lLk!46LcPYCn@0bt&R|)lSmQ}n5lKT9i1p*{ zoSVRz@z_Y4uc!rDoyS-03&U`7rb+RNs39E+uK?VIh34^ohIZ=-)o%*n`LNj$iVTCN zL`Wn=h|GE*5b47a(HCwqA&SG58s9D_D!y0HJGFU>&Ns52pb8>tT^Yq;{i*g zBmon-nQ>)Y=eZoOi}+9&vtq;H{1;oO2%QY!LV1ar{#_--m++T@l3i$Khs;ZFz0FQV zEFy`~7T2+AtW8ErsqQ2GXkZfzRd=cxxm)zF>4-GS6Q8Xo>AUS=uB=*_|0o? z_hGJ;QTWf~4}OM>L#2szsf=$w7+-V(GL9c}hd-5#eSJa?M@`Z=TO=#L;FO9+!p+%!OS)@}FhD)M;6SzYOz&tNUoH;NN%4t#W%7j%UDeaNCSBFEeJ*5R<~@Bq0Be zx#@h48J;|em85vG^q_|&O${i<1+edCGltWDdB9%EAgkelBB`|92EGKb_%yH=O5TgQ zCn6jY<3ovaYiTe7gS{(x+5!KHJ$&Y<%0pDGWb%WYUSmQ1UX};TY;n$9Z@Behop9u* z-3|}5*%GyzfG18i_125XWp8yH`uDY=Z{N_}Q0up3ytc^b+5=2hrXjrrVgca&Ve!n^ z@Qh3+C-$*ODnu_~2JxNIN?qa(6!6{y$=*SEFRx=b{7Krq}}^+fQKQt5bEEO8(g~w?O-RR5yRVjd=>XxD7jo@_aYW9^wx`3Q(tR$Vnx^ko81KCRYLp!8{j52lf8x*R#Wpw~@wM#-F#Likp zGq*lp&dH0%f8M>am;ioElJfWtf9Wj8+o5ZheLmz)R(?)XmATL zqf;S)(CP7k99_nI|FzFgcPbAg$rGtd(R=nS@Wbw?r&ej4XS@ZX%4EqO_+64{cXugR zh=oxTwj)TKGbAb12z!MA+U=?jzHRt;fbSpkBEzuJau zB1cya*cZgleV}=+&zh6@=2@MJotz}39Y&&S!(p$tM-5LYon+oL&x6@v_nzH6o_+L1 zyrKvidy+gFlQin(L`Ktlz&$oWP0dUt-{hOyb1lX9`I8sa-XeeuCb8KPrgo?im4m#N zripq4I|YCtJK+7yvhyN?;1>LQQgBFh5qa236zWcr`CVYr3xO2!^vxDH=DoNK4-VQc97 z*&>(#Cs_kF1qFZ+c(MoaWlOpgSIWvs^~87xr9WXqaI8TJVQG(y(;*IVxWM$CC0fIw zj+X<^tF&BI4^J<*QfKW=yI98I(4FjoiVw`EQD{nMf^elx5aWa<7Uz4+UAi}+mSwp- z&4U6}2BE%M+C2=_Ch9Fth}|wc=jDBq_RA^~j2afRo&h}BEV6wgq-{&qx&C5EcAHh- zH@^fCZtA;j_;qS!4TeA)anzp4V7C15$432zfAk&i*YhavLiB2&(EgoTI>UUGnE(_r zPy@`b_`GL&P_My9-@%r^mY`>(5AAi_DBB_gcB(ATYO22=183k=%-zCmWQ=0ta({h1 zmAe+`5AcO~oAAi2i9g%d|ArUHT5deW-mbHz$Z82Rd%LvDC!FN^mQAX0jpb5dIB7d2 zS-F+q7^z34$L2Ave=WWyltM2hncu-QEgT?tbdf*&o_;S?D1>e%a0{`b5VT?o4PpjkU_I4Jkz&H{k4&z?W7AX##0=%KTSyyI2a{KmqP(1 zZc^Hquw8|wu6qr(`lV{h^RoumEwC{GEK%JkvQJHxUlH=OB2`570$`cyh5H%nMf-!; zwP)vun>?FRY*3K*F`-jmp3cFeLDDW9WJTXe#+s82Zt+0+8Y|(DnQZ}3)p8D72u?i- zJD~|=gJ?E(#=YMsvBVJyMF{%_R)>)@ETla`JdZ!ts~YXZ0>>VUqyDAvP-KA%6gI*I zm=dkxh=iqMnCnwo2)%l1bwNmhfZgS~fj^3ZIW#7*+f?>J;aZ+>=)*!BXKZ{|g=d?G z%6h3v4)0dZr_y*Dx@OS*4;9Ct9`Dyl!BMo2H^;qv-U9dCUqAW(47$JchRE&z&aP@v zN8+05qmL#v{Fz;W&xih29is@;Gy|-T0$2~8_7Q?pW@l08e`^_p-=k&Q**B9Kx@7kQ zWwDFK&W3}0141s};}2vE_@J7i9uW3|)rB!kkFHh+@P}4?A0nchV|+y4IEQn*agMwu z(fz@Swqqf;V|@mL!4`21jSIjZP@?E29!S*xIZA1QaorbZk~9gdwk5FJZq+u%0+3tr zsBmyvt#RXM_3g#pVgb3dYLt+)Zfh3ZJF1q$haWF4WFV8X&Tmu%bkbWi?Lvrc%3LAy z2#mQ{UcO5|YN~@j?GzUCwq<+94^Ed`?*PeGkU!!~s2jGps>`R=)4I)>Cmx#yTm)9F*3^AJ zu{_mzrlk;P&&O*{$fS2#jV)K3m$FW-JiSO(r%(xV@yx)s=hLZa)b G^I3GVOfiq z>O7#jBHBCj{}Ot}d~ruvSeBCx3G?QrscLFs%C%D7$+&TF&a-?aFZx^9>wJKROdVoo zY>&5vm~=i%jAV2CO|wHiP@1@Np$b>SfY4hPdofjMqSQcBS*fqE#*z_cL%nXJYIA0% zlx%FLwcTZ-z8nTV^`M``q=FYwaruDGMv#IYoGTB{>F}^x4|^*@bKaRzizvM~3-vN+ z+DwVu>kYH(sZQwRUQ$MMxjvw(?D0Yro4rR#wnO7;WBssO(TaSZ4r<_#PbKHrVQlWl z3!d4Sg8NKlJdX`}`fmB~-kREx50V4?L5zD?KGFjM!j$_u2)5sp2XusK_7tWRGe@vI zzIbbzXFB9h(^Yyot*IDy zMOMq6z(=rQ3<+nj6dL^F_+xk7OFNl0kUM$F#*w-9V%@1=nv?c&bt{f%taV)AD!n)6 z$9B3smTnfa*#6$(z2YlChEPja4~p)97>KPJzrkIz$W0|TR&xuSoyg9fkNbPN)Vz@8 zmb7TbNqvXCZGVl6X`JD~HJtF!Ati59n;eI5O)JLtT? zhM{)g%O@{dIs+4kE@GB0xM;tu*!i4x{DkJ+S1G5==bVsOD!wX!mz>&=a)x9dX~`C5 z1U{oH73hyyzT4nW{8m7F#G6#>u7(T0d-nb8L)&C{yH0!j3>nd$E?#qu%o?S4$cJTa zAUd<(d}w2Bmm7P!V}~(Fm*jm%Q@hh?TRV^7rq|qLa35Y;dxSx*wBlR$fZ!mF7f3SrrPA3ZtqNGz$SVrjiq1#qeq{DcQ1BYN{9U(t-F0Tnda*S zUA}n-wKHJ43MmX*xFd`B&a}p?l;AtM74ig!_@hFq8QKwc5pj0c@{agiVg(3!7=D*r zH_trf)P)1LAgqjJ0Dytv8oxJZI23j(++nsqLr3|7{t;0U#5doX6&1((x>seJry-Zp zLR(f0w?l#i2$s(;Rd8SYAuSRkKVH?@aCHrM=ijmQ z^AlTJ#M2Xqk1eX_{=>`a43FHB^LNRF-aqxvXWZY|Ttm?zBj&7OxbhVb7EWR zTNkFHjJ5#c!?Nx5X~SCg8^361sch+Zw<0bg@5h5+rsoMf9t3ozYb$BuvT#JN;*Uu= zYyp^4lwIyH;6a5XuHqXU=;!eb5Cwz~~aSAa%@Snqoz@ zJ{3S{jCWL@?@&RdsQS9$Bu4S;=;Z3P_o9*0VBCVL4Shs)Pb|Vn?g*UNJ{}t74=E@r zqRlq|R|o!sH|`>oR!Db@BJ}GhU67R{LURY-sAMKSX=Le-SFc4rHA%R?ry=~rpH~Xy zU6|M(yel~;G$oV}6r(pf(%9fIeG;qmRj912?)&oeXBrs6A|$gH3!!K21~_I0Y+k1& zF4*#t8Bkl63#FGhHn%GsFI$?MpxO%MLQLB00OC-B*z(Dgw*`E7zHDq0$Nh~#rvv0a zAom~8qQ$XvhIFucLQjApYB0S**JpiHu_qXCfl&B)MC(BTg8hLJLDcso`t=90B&9}j zN+g!fB%qafBTB4>!RcIWxT6|9I|xUzF`F*&fe{V|!ckcU2Tvj*lp>O$%6+`Zu6F2l z0BMB3gawYsaJKtS;pfSnVwUaKfUx$*h6hngC=l~&J`FFYX4JY2c@Af9c%DI%@YDhv z?LwS95}p~#B~SEdxN&$1H+Dv-WS=CBrMU3mx$8<$Cc5;ogf_((G)L4_n6T_L5IxBf zf|K=y?SVu-qJ%bwJP{gae0COkk~YK*Kph@)PvZCudEyW`2B{?m&m(%$CkRVPc+4XS zU9feST!YjSg=ZK&$phl%1uYCYpO`y&j${%8F#*~_tSJ|orsT92Jp6ZwN)VS1D};tm zQk!HF9)BQY)IeCb)bv+$^rvu*0|K?%oBv2QW;n4!Um`%jLPX<2K+pplZAgIQ_UHhF z0BmcVpK49x4iO6EL@6owAJbvwc_}0&P|8#!lUO>YoI(-oaxYXuG zA`j@nJd#yWdQt6*iF%6k12xj!29gMX1bK89JS`Kd{g$9^)uD45;t?Y1)V+92+})@s zJ_Y?KDl2MVKEU==K}k1=^hbBN_YTuuoe$Au&sL)M^@Hm1*62I$N01Zg&S1g)lWW}Q zJ;&GY!e4{Q$PPh4KLW-Q(QY3weh}XuyhtRoT1y07K#Hl}@DHXk&%Txk*i3{2tZixq z1%w6)PESQXxEl-+Amm&t(26meVHWI*6r^9ni&E`r z_P}Q^8Si_P3&Qva|IR1a5?0+yb$@RkfB$Ct<~c)1&51d!YBM`K1M&Dv*O4Vm$L|7y zzQ!DMEmBHJ8#t|{x==S!7weaJakZseL#e9m*imyT-OIrVc;68XK7Ej_J^=n8A?m~ z(lHfwfPN}KYqS`ZvR~S6a_vyCm9pcRQSObO&v^osbu~p0zt-WG1MKwS_O#*4rU`Z?NU(M`9bO zYe&>+JGkoyT5R@N24L-&JMNU{dt~v@QH55K0vzm-T?uyjy=4N_M!++{ zakmeUwsBpA;*lx3R^O6n(DrEw#LQcKF~7$MFdU3BgRoxfc4Z|igtKOhnJD+`KB=}jG(rGwvt!3!@EN5Y$W7xQwq{yUxocJb4N zHXYWZI&wYSUWG@2{=ksX`P0Wz^XL?PN$b^f)*pbvLiHi}ZQe!2hsi*`=}CizPgkEI zD7qGeOROtS-0kyAikD5FMmov|@>_HT67 z#9eWFwde_Ein)jYxc32K{c>v-z` zEV0X6J}VqcpA9s+MiLCT>;G9Fg zN4UPJSWlL)+I_vgv>DP@OSV@>-Sr08r;xg4Qptv6E(qa4xfP?pN zQT9Qf1UkpK>zra^hL32=&IMmv!g*RpkJ@%XhT@o=@7_K7{%HsyDCS%IE}zyl>Mb~#~b)Q z@%07F!lj8h)%?p*?#r~X3utV1QvfT3eUzHB#%M^LG*c7Q9P>kE3}WC1YunmJD>uUL zKFP~__-C|mwr~_U8}ckB@}1}CCsd$|VQ!`-c4sek?_}=DjL7VBYgZASE9UhpHVD7b5TaaJIoDbx5Hd$VvZ#xQIh(3EH5wz>P8)h#4ZFMBXU_#1Plpt zPeB*FO)+Y{sBG_4!zU#rK|uH@ej`m;&9;-7_XUWF-phu8PX5u0ClP^`faP$dVscjR zPPbxFYo#8G*x(Gq8}Dy;>aa?XP}tDee`$LFO_wOZ@sS}*65zjoZGTuI{{0DYndmdtY>0sDv`$!arm|EF5gv)PGl5ZHo%Z0~xsM$pu8;h6Qu# z!G@oxU;_d0Vv7-Mz}u#RdW!|zOC<*CZ^)M+rX9QT(I zbq5oSq}ZW?x*!G{sDQRRD1RK%P&9>w68P8bdjtpwp}#8*UYCZflKg_d-y#14zQ&6r zSPTwvq6Obc{;L)KUP?p!DSpFYfM~HOIKPkTA8i!E&g5_3foi)rP)^jpfZyYZnEwkp zE)`c)0(-v#C*hU-+ckJy8Zu1(i*&oo^haf^@2HEA!4>hrfPfHr2KQhB!^MDVhp>OZ z9d6IMD!|_Fz~NcK&)^%Jzr3^M5@G+hO8s6Ki9pDGT&PW+Ul{QUSRi)+?tff=am>NW z0k@(8JOl*aGuM0k|Iq68aZ&&6OTD@o?}77R$MeGfAQZ|l{^JS8#J;eJ1 z{!C%|85~va7aY()`ky0trZD}CBCiOhPy!K;kpJrAM^sP*#=z?#dbqy_hCSFsYXwX? z;s%!wgEtgyfE07Wzq|VO{GS7k)DHN0Ug&obc+fNaCGhiq3p&K51jzo`@cSkA^Y?DR z=wEU_|Dq`z{ulkPhR^>E+kZOvf5m_P-fd}!!T;0EpHe_}&$NL4+ZF#SCGzhcUc3B1 zD*FE`EaCu!+{XFi3;iVm0ST5V0e9lR&TxM(r6JJnK+rM+uxAr_e!>4QNdM24eil>ujAByt z59MNw;E#iT7UB1dLRI^lf(5Q+h#mpZaUBuLc;uHcVIA{-rg9tBai9>!e-#TGp#V?U z|7_c56N=Avb2;&wg7r@u8E#NPNiX~|w^@+zU_L(p3)n6plO*r6;4c7k< lf8PcAznehMh%YvN6Y;QNz$bhN2!8M@7YlrTKKg6x{{#N5%QpZ3 delta 40255 zcmZ6yV{qV2^zRwlwrv{|+qP{xzlm+zww;M>+s4G4OlF_`->uzyx2j)sSD$wW-%r;$ zefa=BJO>V`EC&t&mja4_nS_T93YWy>0{TBEWC#!t5GPkFW^j=I&tenve_f_PfD&TE zfq=lk{AVNrl18~)-USK*f(s4;!knU#iIp<;@gh6OxJ_@w?YlCiNH`zc2M)v1`#LBlR*>wxjtqEW{AUv-^>%_Gy*^7-3w3s(%HY-OxkD8e)TeT2S z@O3PN4+8GfSf_}rnrSeG$6Xx%sO(R1s4Y^{OOacYVEg=r8$VFDUG`Ywv*;*rr};q@ zg~uCIr|C~BJSBa;D3@0?M=@1;DP>^QcImZsN5j6VX?$r!lZL}vtNty82GjkqK~*gd zXRl9nQfc0FjhmBbYLBps zZ4%8pB4)qoZ40a}R3w}fNz{BVcSTEl-Rb( zuUBnA8ES|jK|m-{NIkfKZ8dZ)oPUIFn~e;StaMSqO`%C|KtDgBmWq z#G;DI^Hu zd91CjSsA#&74B+YitO)blk^DRu^I;m+ZwwkEon<+9p`NlAv7kdQ+8vRvumS%aTi|c z9?ePWj6Io5Te^e5!DUZ-d-KrkGw-zP6jI-eb$6Br$M)eXk60l?Jpy>t^%di7d^#6L zOXeJ3yCc@PV6B=%oSBhb#8ost@mw2TC+HgG7K9fu*))($?20ch1Ts!3FYz9KU*Ah_zg%>xq zqeEGKuXN3(+Hi$Te~fF|FTs+!PE#~@SFWb)bNYD4EDP9J(kq(Y!Qww&`)mVw~&!ZFlJziS5}PF0l@2J^%Hjy40SL0iM{4_3n5 z84j3YYV(@9E>|bCn*ygrD?VWNSFEGc4OZP&)qn(y4&)$>4(+Z=eV5q7HO`9r;ra)S z3H9snIAYWhKu`1__YZ@D?;d`Z-(W`AA4CQd-;)n7(9^Lz3pD^XkL}usr#CivlDZUf z=>|*I%}P}Hd8=a&FISpQi)T@LtCCu^K|jqHXB3k5#hYEME5>rJ`7~103zF2s9@3{) z_{jm)6GhLun|swKUugXJsIf(0@s2t4G; zw%iQ^BjDmCX-3D*^^XiB5u?obH$ z&HRJeF8QYOE&jfbryDX0vL@+GDJADM^44mP-ZdMvPV~W&x(JwGt^CLQTrU>?35cbN z{Cw>xRJ$m3V;0$sKzc4J2;_++k4KPbQ2pX|L7|N96J6XVM?~Ez#s0y_~`M*hI8912? z|9_IY0sWs~{wI+{DZYL*z^)p;2FAZ#doK8CC{<~i)b^k4oDj0rnysaE4nfw|G-=8x zb$l5oT=qRTN0#7!@jnmLj6RE%6pj2ZV_{aaUKOl9vEN^c+4)M2Lx_*ZW7*@+4Tq#E zelD|S<@&wl`pw;W3B8^T3KQ>x-y`g?p^n49SP#O|LX}NqQYfw+0;y5U-c6vexk=eA zRC6Ykpu;$L86b0{vot~&1sgg*1k71Kh@ykvbR zPhpUU6;D%{CYAKCQmH;9IPs2N&Kr8Jgb5i;pN5;*X|j9u5JMXEe%Ss^)_$u~W4MRd zVfZYK1cwE!2K+p;fU{5+e9(CCO_nJPvJd6aR%(ZVOA((i{g;f;bz(X19cCB%Xn>y3 zdg`I^Ts@0BQVf0X6$HP8Wuq%oofuMsIs|X-R@{2gzeve4nN-_$(SZDc(GTm(b9Nf3lM&NRJoUH z4BHl$nnKH~53`p6&M;jq*w+=YrcL`Lpjlvx{jwLz$?tOKed4898^(T~63an-({#0n z)@MXvw9Jzm!f+tZBfCi<`kJogKx2~HPA+}%h37BY!cd@o>q2xN*4Bcf2{9i7frN0= z759B~7%T<@Krn_HvzKFqD6iWYN(;Wc^hlTHfxm_DCv~l8se-!N(Re-vu|zzV^*%dn z-DPtKztjFeA$;cF1s`Q!6t|F{21XTMUjpkMg08A(Wx#kxm|=C>Z#IG&#AMNuhe!CM z+iK?+vgMXhXU)9ykR+c-L(+SP4*8IdiM(o@U!r+Rz*#$AJt0ZOyXRn}nX9!_ZhPGd z)w#cu46eCtTuwMEQ-o;F`T8lssjF*vt543>T9e%Df<%W2!3Z{9_UXvN%Q{0u_*!Bv zzV5L3egZfJ^3pQHC8zFj6=tLQ9h+!XzlAle1MVUJR85LGy?b&&${lv)c!u?eR_HV5 zVNl={K)lOSTEE>nWMNC4B-UE7S$o)*022^2SpS)GXR351W?e8S@6i^nRhZK21FxbQ zo=r|Xn3;a2gqkA9jC)ti1U4zFR*N)5@_}gZx{tQzmAd;D`V9VgPp)`CJ<24J3+oWF zOOh|D+JrBTtRrT-UNiTyXc&Q&(3Y6J*MN5lKyyxQW@cU87>WeUxSv5ihX%q+Me6Vr z{fYojb`v!UlW8@JxGN?Ny;K*ipxcvNF8qcqX21P}!@Jv8IRF=5s4jbexRU;sm-rio z_e39m$ld%;hB{CR3NVHMs?_7ryClg9%EdjJGu)cCVf%af}^xSZ$o zf3E)USwvKswhL}f-h0rNx$a-|8ICEr+~Tj>KAIQF_kl(O0r zIVZTf@}DIRHP$InO6BDni!j2SAp_m?6`<1U8Z5(s$q&gfR@Wg!ck)qZqQb6mq9um{tEdfcjs^@n%3_>4 zi^arbE7}gniM%9h@bYJQ`r4`a+ayE@f&%Smtm=C?;7B3oUtQ1Y?h}xt^{B^NNo#rU zj$+|#>AgqJ%{&w>Gp$R9!;PpRhdLobqNa3V6K)ItD^nx#7rWzB)Ow5SShdCghgiki z3>lP^543|GIKu5Y2<}K{K0BJM_Gy4SQ5RTlK`0c5taqdo7?E7&AUdoPDk=-T3DOfn z+^L2w&cHh;i{E=EYB`TS*cFDp;wb zia0w`7S@*Ejt+mcrE2?(Yt^S{C)NEO;{a03;FOEZM9nNI4Rhk5DJFtGAN3vy-N(w!5RUCP#D3L`r-jq{U9sIA zA$|IQzdrg~VSW+s`g=_mu0jsa-rz>0NBleQpu~a2#LzIu*hlXU56|csEV-TGiqWoS zXa=RML!XNZ*>UWQ>Mpz0k$**YAg{*c;zUKrDICHa8<%?)ondHB4zn{94@?5#_hU4Z zXGaFa5bDv6iICiQFZQd54x`F}|1+m3Kqdo^kwHMTNkBkI|L>dvn9S@=+}!GPVZHIk zJpvOTn&g~)M9?TNLM;d}kQb6YlGE*ziYcUf#S>F$b&|CPl0zX)4@X5Y@_6EJC!%g? zJngAna?2fa{#!grxiF~`+p-B0b-P@bsBhBS&(aC43QFsJ$&LRIutL0^aX)1TrG@UZU%F~?7G*7x>p!i-v(Qu~i5qOqhQro;W3Db~1a227|+9+`ji)df$P+z-0kzG`; z(r3))6jiMoGR&Nf5pU7vkP}y<&Mqs~p~5Q|n#MFNdXz=JJ=e7l)vD^<5~KEsn+i;G zQuV1T+yf9Q9{G?FAAQrfy)UkNRZe}wxM5X8rsC+hX`gY!as&wuu+p4R*|_L?B@Xp8 z_NpJb(mHB-#ZL2v#tcOLYNj0M_RAj~)7~omd#1c9d$kzswK~NP1LSpCh`4LxGxO!~ zWXd`J#G(pfw!CGbKV()(eKNSMuq98{TQ6}9OaLd4mpdnWse+axx7OMn3Me(uen@Z6 zM7O-0E1Se{udhGru8IEE?x8IomoSoUM~_KuXVUS{crUFTza|pN;nI1rX|y8$-rwH@ z=r9DrQK99cuA-`{8-Bh00IX=-tB*;{Gom90yW{ur^XKA0t$8?6Zhqeo67Z}N-dw?j z^8u_a%%P+BcA5Tv^Zed0EsR~qXBJhO5-<(QIk(tsFOxDC*-S2@LQUG-CuOv&jdw~C z@a2h~!x89lIgjNzqkO)tnW)Yse}|aAIV^gw>(oX2LFnMyQ}O zwsVv0$XtglpGK;OTyVLDKIga+S%MX%QjmnteUm=;U06bC+kTTUhEOH4NRDcPr&^PU zNsGgp?8))_qMA-W2I4{hn+wepEyk{yFT`cL@yHheUR7pzi{F+pyJ1%l#Du={EkAFHQR{9PN_~}eUEsfo9^U^4M*!38m7{i zSoB=Jd6|eSWVQjm=rSz7$$LDg70s$<;26?J%b(PY3MEE7?kFDPMI1Yq6<+(!xX$O>xl^)-raJjjR%T-g+_T z5a4F#!bJS?r;ft74|Ox6NQl&B5i$h>Q@WNh`?EIZt-ZDs{z=BGi!f^|iOAnFam97R zicV^nKW>Ff&T4HpITK{WPT`iWHHlCsT6-fHfQ%K?BfHT4qMO%^83R{O=>zD05GBX* zKjQ4N)hy^_cZibtXXO5v=+3kSMLtxQ?}!`BrC!5I&^&A}J3m6hUxf%&%*66MW<;T$ z4zh6xP=O^9!Dl-{b#YfWhe8@yn^Qcsc4W^AI&x*tQrfW#Q{l)@%xWGAr_uZ&juiza z8ZVi!0J$hovkoIK|B$a2rF z*LsWQ8!H`X1J3_Q$_U6#2vcPr+ggxM%9(kj*+(p$_60@pksE!8R&L-o9>^aV-Kh zgTh7Be|_M=hi*cL-fZn%RLycwHC5kh$Ix>Z-g6^ovb{hhRwj17?3E;X_T9v3`9>0W00{cmPirBX-p9fVL5l1M zc6uR*eWstmKht!`|G3coFxfjP{=0OT{X#BUB*zSD~w^}!7g*&^qx|Z~f z+r2);2Rj}h3K9fyF`xro&uMis;Ot?^BP6FXV@~462Y4^glU$=UYSk*W*D8HpuTyWI z>?`;f#0gIpfaPL}WFPeJI5@3}l62#~73nooUi=_ZEH?Ebvj6q&K}sBZ~uT(Sm~N)Xn#K?}ATq_$@(5 zr1oB0A@EaasuIK21AZj7#;!Ty`v!AMFpvtZE%}iQK(MgLSps z_4DGu%@yQ*V*UsKi*QY6pc4%bgw@p6xG|0G0Dlx5URW&l)*j9MA7KqUy>R>MdW|T zPxca`dbruCXj9rc`lDyi zL>N5}Lx0VWB`cDI*>dOd8+93P{gzh~bMp%^r69|LOI0t91) z8%GxYe#FC{$FaNd_wU{ZXqLc2;~UHdvkSjb{49T=8-WMaunOqjgoTL3K!_3%I)EHI z{}eH9XGK85Unbf|oh`l4fi%srXnetLf@SIWS>+?IL!Zuc6TMp2T#1H0jgEO1@c7yZPqp~ zwVb3Exa)OA=uP7_EEIDvI(jHACm-5Wu zeWr5MJ$Bi04=r%cE#JA7U6!vkZ~6L4IB1Mo-+C&2#7938ruDNGWS8eebYx@*D2-w2 zW|Y4alv6SFr~A6EG# zr^%K#mGls=>}ftPt(>WqU$ooRt&WD#h(o?9Tam9M0)a5J-Z=WZ(i=h)mcpDWFgE-@IdH`}B3S$|R`_jjI00_L z3YJ|7a(6Hf0Ikg*&c>H}i8a<*^1^4vKTOgld+Y*{4;&xunXo&fUk)n(nZ7>(A~0v{ zk}!HW8=|4a&j(!xARx)

$|chPVR@j{Ne$@@WnWOc#pdglpYbnqRa^G=aT%hc%Yu zeaQ7WQv&S z-Bq<`n10M5tTIoGCSRX*XE}4X-yKG@*&4QX9KS>QdOcc?HyX#d?b+Z|_!H3wjaPATz^QW< zXl!5w`X~SaObxeUZBT*fLl4?DUC)zRKmhCkn4pu!i8AUm!+V1NwN#1|2$6y|u52x5h~<_a*4PTB zwZbROhRfpidY0V55XOH=QZt*L?wXbvb}pSCneNmv;npvM38FTjLG&fWlf3`s2_+t2 zKQ4g+yu+0h>84C@U-5smdKRU|7b<_M;F!_GK`Ub+7qf)^`W+0N9%Pnn7!(Qucc<~J zU&v6het}2E14V3QjeK5XAc$d3d=;ndIDGong1O7+?_HxZ$c3abZz2~p>E#B72eu(E86|crO)*r{Ah21 z(zfHUdORg-wmUIpGP^qQ2|xwyh1fO{oeh9=_Tgn#<2tY1XK2C2X43w$dkicBO%4>y zpv4>!E~T=$lacQiapLm~#FrH}&A*Qy8e+Q3Y~`Y>ec20XUp*BhC$iYf^zQ6C=B!2l zSRVH=@jMSzu<#_aL&JB4o-qpeIcmAW!|iqytW(bxFr$rf-IRQAJq^RcmOHKcg5`E= zDudUZrS((%_%m52HgoWyhnyxRtYJo+ruuwbaG}#iv3P1 zp7+Gi#wUe)DwfkAFQ?y%y7IW4zO~N#aEW}r4(UNOqafk(i%i+`t3*dzPVVh_cj^v! z>F6lBz~(jX0RhPQw`0h_US+ho8gEp?n{l}>@2X%w^%dDMUH!xuX0->`UeQ5%jT!gJ z2Gs}K9eR_ylwrG*dtJ=8V-GmPyK($4?-IBmZd&h_=rHe?Xh`px;EpP72GHN*;BZ9G z$G3Dt-VxDWM+T4AyRO~|1bf%x62eSXl_P&nzW&k)0Y8zeCycd6VTe>8SR^t1r3WoA zuU#ZY4LYyJBTmB;-XByTiBk%QGhzyqA7uQi;R?pAFa*eWh3QFUz6pewBbY1S$@)u= zrr)E38>%hhN_Bg^d3xNT!H6qfRJ@dT<`Y<7HPvam z8ix$50GjMs%`c^WMhO*&LCTY_?XP?3_0oQNNEJ-e%~nQQeopax4L^08t4z%pa9g-83$>rZi^^|%#3g6NL7ql&Rc+~6FalP)DK7jz`tEUQX1x+mEgZN-oNrz%T| z0Ja@*+ARlqYYUkp5`whFxvvE25reCdLi$qRT4cFH%2!P9J2Hl=Oh-pS99}&yT>sp` z#oxc9jwqe9`Z$S0S+R9jvaV@{$|HNq1an)pq91JKVR9Q^E|z60j2Wmh!;V`W9Wi#n zu4mXNesh!#f9>ap9z!3|95OcX*43)Gfa7;A_PiUX4uUi6zYCWp9OG#37znNA{&YkCc9oK$M=^%M<v3Et~jFb@I2<1=rHpGJtC`s#4w7Y4Np)7 z-FsX3{y_(ro50Lf@+l9|`JpHFobqYfQqeE^=L4L7j=MzmGG(z4En~_4zcPl2A+7{7 zJDN4i>=jFYWjfc303!tutzQDgfXweX%;QtVp>Lf*BaSajVENt)cLzAg4hN3#rib%p z;O&xrXbh$Uhx@MkW^msnhb~`QMI@vn7!s*AL)254Dn+mo?^#ORJR;94-ikL0)VJD0 z6O3p^>b9b3y6^F7J__ov3dU|V`SRJGmHUNBu;DBiY3GiS-*AN7b;@f1HR;;Ng3Gjg zXtBM__;V_1wNY-hkxF93?zke3iXOU}L>q=yNu*77CFXz6VjEMwrCJVdp@ZzqAiL(E_PK2Ojap$H%gcM8Dvq220xmznGc4jW41^Al2d-gwOQ{Sffi;Tw#nfQ!8Ky6 zFnp@zRJs|43%>a^AMi-vw|-SlWCSf+Wrc2Sko%DI7J60saN6HTZ+j94R$lCkl}=SN zb7mSZzDM)A2m36a?cAdQQ96qYjX5zB$jt?14(!a|l#+Y= zOP(>cI6=;=5CM(ON5AFpy-AQTyhJ-tYH4kXQf$w|ps-&umyErE{bk!2ixkiI@Bu?e zk6L=4daP6>sh_4inhVcjhBn8vLv@?^c7~MeNp3Oc8s!_1aYTuu4%I2R7#UhEeC1~& zUgb+8uBy5n-UPduT4!hS-A&I-y_EFkn`P7d>6W-+E(UwpwHP1k5!9Fl%rl>p4rN-l zV^xZE9tstUT+&u$TWJ>iq4ae7yFfmc~TVJAM#{A_aUhJdSDmCKmN zAwt!d(uc4Pf=rRD>da&AFnULfIzvv&Dfh%5$2Lq=k`AVRgkk_JS{4X12%+PnTqJh0J4+ z%8IL1VV8h5)7u(kKpd{TQ$5aLtR-b-qAqK6mU$MCHh6;&EK)yX^9ucUT8c$W@r+Z8 z34id*lCzR4-c-b>(G?@u?4bOIH4b-zsfp2oaI9W?A=2^! zl)4$Mr3vnt8Sk1gBwthVUT<2w_U!qrE>0(+RTGht82j%r$y}>gCS&;e!c~{u#A_mg zp*%(4M)E^|Pu@$DEQI_kTNE}U(jCVl^@+rtfP9^Z0#!oTATU)QDxRGHDo^OA2u-R7 zbVGau?)cNn5riVzE!ay8`JgieAukx!uaDOWNnY7n+9ftVSB%5{u^+=Mcv6k2LNm$f zXGvp(stTh&9V*43(--`0R^+kau+n}$d91JkvT4lywF7ONt<|Pq>0SSruqfB#drzue zzraK{cq7vp_8FuO5z?QWBnoL8*+TRpnyPW?`H>wpk>VQ`8FotC^HzGe3oRS@LO5}2 zXAjy5k&SK5F|PllBOSliU@Q`BBo2=KVbMba&+*S_>T0NMvVzbHcNy2yo7}_Nc>eh#g0{O-4XX!=z*)2Et{SB?BSwEe%vwK$G`Qbz9<&(MQ&LuI2SU`y zKOy6Ssu9|EV=f&}crPak)v5)9nJ&i!ntPSs)M|rxXWCr6n4Db~Fm<6=BTV0mXQOor zFx3R3Z^avmDot)^;^!s>IFZAE&lY_w>55!`XQW7Mg4n>BRu$h9OcVYcNgLn)XKg4SU!o z|JjXVIN!WSavbN4-q71c@OH!~|E`(eUq_(Q+eg6K8_A_( zkjHA95%U#rsF9~u-Zt5MklxWW3&+B3S~XW~ArCOz2eOr9zYa|~1;wta+pc|=V1s0p zvWl`Fq>)iGT})y4#C&Z>WrI%3T3d5yr8p7Fw@Y)nTOfKG1Tiz-|2qLDCkfse+*L?B~o}XwTQ$3d{9>TL_Z8-XD(^G1# z%f=XR^t8ALob2v@$((%a0K!DJ!s|K4_=^g&q9UiAD=V-w1s^y0ckXpf$0xv zly=VvX*A9Yxj(>7DNuZ%=dC{2!Ag6`57!@a2Mdv%TzyOPE)@7f@)YDpca6>=GUb=_ zU_#b4^?Oxl0qr|)V9|jxrkH87j4*5BIYjKU)0~i0NPW)_ z`%CK{`b%Xn_?TH-A1pJpQj^|cq*`IRN-?M7deT<5(0067kpnigo0HQbi;?1` z!T;db;T>H5Q0c&u&7DVuf|QcAn&_sMduZOx$!QBEy$$nN>_A#eu;js~ocy9bu+UN# zGv#T*r@WYDXzj&8r=Onplx(MxpOJ2;Pg}%gYgc`IRzDD)oYrn0m#b3XywKF(>C zF;(&U_@me-BYPpb#rc!Vc#|qEX)GJf!fQ+w>udzamBC5oU@KeI;#X&2#&uOY!GgY9 zl_TU!s@1t@v)i@t6x&jKyPQWjFu`ieQC)UXtSh6X;XkLPe8_sHoYi8RqvJ4aZwt@+ z!NI$yF{8(qUgi^HmKAZ6Z$>LsQnORB>G=)W6r*^y@WH{EWc6aZ*6TBu4Nuxs8HyyT znrsDi=;K0Ls{AvAm5Byp#)5K1XK>x z|6tM}?DI|jM>OTPG#k1O__kZTQEntByvx!0@IvBi503r&gL@lY4DON4z^MqUDP&N3 zb|ukI-b-I0hkr<^zMP>5KOqD`Pxnp>q5O+-dpTs{_EO>G^_jud&zF0VyEq|`|K9(| zv0@QqibMr4F!+ljZZM29b%i&6Py_#G{Gb6IdJLr!9(o+57C+({08vRlw6qNwL9QKP zT+v*_llcMN4dOMDeIYWa1hz#k*sq6bENJnl6Z&~gB(4MXG$;C-SgMO#jV;rt>t{9s zR{U~LhGqb+K3DY13$A*6aTXQ?voC%Om9vl4Fa%ZtnYh7f)v~teUrgN4s6d+n>efdg z>2LJ=KU~W@!a)`Wz_b*rKn?74g}n7c#9tj1Q!p({8TtmY;4y@Hs$zudzF9kM{|x|Y z{(7YIRN74mM)H8!QaF|BlSH;Zlz@(fx-X6_J z7FJUMGZd6dDD)^L2pNie9&TP`#~4>b)fj%%)IOh=%NOJESD@RJfLmTPGK7pY`I+_3 zi`9d>-sN9}O2}&H9cO4o+eH!bp8yVgir#V9ee%%9P*j;Ma{R-IK1hjr)@)<#y z%0RqZQ&>kV;Myf-BOIxT(BO!~GX~-{x!lpDL~18N@QA>7lzfAFuEa310+nIm4XD^JKvg{)=65vsWo z3uX8gX@=prNe6NG;crUg;uOiW)^^Z6BksD_AK%3P=j_S`LNbLv*nj^rpg_Pt82?`! z(SJn;(ozkS&_yu9_iZ{1x_3nx14)Yg<@D(E;IdIhab#ktB$!zg?j5zmn;ZX5IM#fV zJ9RFI7cY*;F@LFyvA4+S$s%$n%+GA*z46{{X6*_Cz!#YE5IMNZiG{YJGR?&Ok8*mx zXjgsC#2+%_cp)k;@BQ?KT(-d`t^OnXZqqh^HZy^iKsh}0j>~rb23G%kO)D9Ct+P*` z?QN?-g<+Y7Z)fzNzs8&1jz|FU8#9LL=TnFcwvbe{$V5(DVOC+O zVTO3Ci|u%S)Xi&eRz|Nq9hAJCKJshyIqiRIcAifN^j5CV$zKOBS@HL}k zVx)c+o#qlkSUo?JA?y)AFNE=|3OQl72=Ls$At>j!R0BNNES5HGS_Sncns>q5$w;Xj zkJ6)5^w~|wEQWq8jbQw~(zep>E+$6@&^sB2#lqd!w{!Z)O*(L>{w1j7s1G9aslfU^ zkWENKb4*L~b5PMxdljO~Zt`(tgN`YUc09>~+LRWGsohDpjy;;j?dTx(O8|tV6ZUkXK@JT> zZ_SCAq}vS{qxoaOTpUZvh!MBUUnErQf06AQy+LEZM;2?M?z}vE>A&MAERIsqm_4|9 zv}Tylq5tG+utv0Q1$>&LhH_mI{Qn~L#1qW91Oo(wi4g>Z)FNb3D^n@EK~S*-k&u|;REyj=RNGtUeDW(+x(;OdA{Fy#J{dP z98|``U(hzp=W=Q!vjFard{VhHL3Y&=#^iY`Nt&zl@g>nwfdEb^$$Vl{*3t~zqrH$? zwsYe7{SFD?K}%^?g^NpK`MrOUQK5^>Pq`3x>`xm@PmKD|m0>`U=-v)ZL)Y#Xds zaDczm%qr3L!(gcu;?ifY?7=Z^pRoZO;?h4*DUs_j49^Iobap6!93Rv%ErH8M({_ib z^w!>X2lYhpH;_z)R+_iu=91{?Ml;}df}3V(k!VJM>`XSX>p#iU&O=ML392~xq>5T8 zb-YL75v9@7=_JrM=U3*_Fm?y88kL$MG~rzkqc5g5eNxDl9RYg)l)W_XwiF zQtzOa^%rVfgZUEvk<50aBT};WXS(;t8jmthmMTALnsTLba;{G00!11$mv8+8=z4bJq6`i1k=0YXmv|iF$ zs=rf5u{=8AtIKlzR(~5PMOYWbtif3FDSfsYYx!4YWj~XZ@^Q_=E>Cxk&8AG9fYS^^ zZSCaLHJPSnrAeMg+k3rqhyQT;K-4DVax$I~Vux`fBW9S78$`YyTe4i0-ZRU{DU@*y zz`43e+BmaqFnw~#=C?>34p$Cvpg=TT2qEB-myKdT`(usF^oN~Ve!2LFF@xw zfmNYzrIm~@pbKj!TN*^?JYmVVcB$SQkU4fYbaV`ck?1nkwxZgumF_q;bMhMrZcicn z#!1lRluX49M!H6f!a?BnFdumKunMLUuszC7GHo!Zdl-*cm5*pZqouiRU@4R(~ZV0xwMs+!D!D zFOqAQ@-E?W-=v0(V&{df>(se{*nBy*K~Pl#GA#C^NMS;8Sb9PaJR|rN$z*Voj{T^IS%3WQiSJ0w#`Knt;)g$_aHAoKz^Z_BW4!!()kl%QJ@t71x z-y{nkvg%#ZuR#lNJL>vt(Nlw1+u(F+4J9e+2EjK8NC%Tmnt3aX*h(6KXqe_`*UZ&% zHCFbVEkRiZryklrG40$bbTMP~zkehP7d=peuC|MN87Y>V>S`Zl5AImF6+*T6U}OrY zu$n9=m*;g@FOrojuGcZOFn+~i@`benv%!`59ax^ADb zuGf_a4VcvD9!vHqhi$5Ww?;lvgTZjheEQ^_K-gqOj?I&A22w+_tfo%Dmh=Q-#e{tw0l1g<+S$E;?Hh7(edMt zsmUvEp3|XTk!XE*v(0V>33~j(VL=MzVw6)PsdJzIr(j9d^q!ENK>w-MRJ+KeVd}hJM@6@_dofMi8l4 zPTI{Ph!hfD1_mdUth|kOJ$QP{Mrp5&!k7mg#MGwm>)SajJ7P?{ElW9yuE^T1i~)a+ z)oHR~wR#8~nY#$2C=aU|OeHSoy;$gVE7ocng625K(Y=l#TznDsqrS|t^tYEEc~V$f zUZcQvM<`<=SncCYvwc!PUVj<*s+`|F4D$g-dqhFtmyl6W#>@n$CX;Od~%)rFFU z{BCsjZ62d&G(H0K8O-Y+5s{qEKI-S=IEQ~iK>~qlKZ|h;*UjEM);wTD39h7~^y2}h ztU0Y@2etcIpbP%DN|- zAhj!UP(Ba;HxZNjFd=LWAH&BvHlA?B)6)?SW0F3O)M&*6Jm0SgW}C0(xrX~1zJJL; z*dahvA+++b{)L|hO#c#-xza_we75@Tn*&g!zLWl?Hr;17E1a)Wy!_SK5pZio=SrwN zFF!3m+#zlHXEZ_xtX>j7E9LzgGXjHC%Kgg1$~7enZRTZX7>rGJnY<((V;9n@pYlzy z8Pg;9>E@EGr+DVPwPr!-n|r)JlC+9o`wXx?unwv4wlOVZbf~K5ft21b*;pr5wuM(lo95(A4bRL&~AJ zZ(gU-?|Bw+E!kGo=K4WiaxYQ0;;px&4?%ir{X~W4?x{u^g6)*lf95#_!yrJ0Oaq`E zxU4%jPCpBuDc1Q73u_JT+j`p79b;8@Z^Ieeu=xlkKDLUZD15(eFC*{MQ>Q#szb6=~ z>V?O*qY#HXeuPmX*5f=L3#{6B(b`XEoAUF*&C^uj%N6wTRP!`>I-Y>?Cnbr+P+Xhs zaw*n*BejuNrYz%F_KfRpXnfhv_5<3suWtIId@oOV+_;5N2~6sLwINzo-Vja+yaS+{}o>;#lL0qLdlsU;Ii2H3xRV6RXoT_~B_ zQ)3X$zlSCjIqN3HIV^I}?n=r}#$qZe!Q8-a>NB=J0t zM?WzJR@|W#+y`-jHk*|sOWUaFu&C(pQf=LlS&x2ki{Xl-#r3(lfsWj3QFc6UmXlofPpfP|R<3@|+11l}Q~QL; z+0X4&sxmidl`e;q$~SQhaT3VlbhXi6UqA`6ygQ3WH|`eM6{R9X=rWuRlXdOmd=#}; z%}ClwZJhb6#0}9yfj>UBY{>g2xGk^;TQp`mhg&4xKGt`Q2v6cX10HMWwANAA{iuT* z8=$viWZbNo)6kL>DdhCp>8NP>`)e|8UduR61 zML^KwJ5GV0ZEFyMX+evgDV2bb6c9!oACHSyQ*#XdFinxL*fVVIADNS*6j=YUkCsro zV3?#!G?ru8-=As$qLaD#PEjAENFFIz>b+3Q?-sL04~4E>UHH93GYiqk;Aff-KiMo% zGsk32pJzoo57fOyw*6?zp2qg0tpwgEgFk}Uii_zE5MOD=X(7w%@=AV5in+%oHlp#Q z1~`M8R2N}4Atf>@IoUa=O^p9dwfiG4L_%0y@cCfr06otFXh(i}G_b?=4Ydn3WBqe% ze(L-uQ=0V@qx1DtZSu+J$TmG_zpW5)C#D-n{0UQr_jkT*5L;KVkG~n4r<%t-37V@1ZO#VStkVsF(Md=!(RPED*^ys&sm_Ux^C4P~^ z1khCZlc(AX0QQskx_-k_@fUnZXUJDEA=hu`%I5fGd2>(u!dv2<1F=swd-G9B-67uE zoBO_eN|V2@?!u!|E*v?qz%#`-j}KIhmw$wt@D(ai(U1j?joLgAbEFit1evi|Vm&dc z^U2IKRpO2zek-Cr&dQw?tDP0v0X4;K!wPm2)}4n$02tLNuKlH8fszKZ)R~=NPo)^!v)%>;TK`@28baZ2d6&`h~T$YqdopE|4a61@|Yo|MrH?tWUFC zC|{s8(h@kpTKr<3tW=8TbaA$(aJEDqD5msk=Ms7WRD?S?A+KChD`i)*nN0mDpKS4= zM@SxT0MRcNkJ%L!PeytJ9zkW8-<{9OmQkezuT&}c#do~SN&%D3w4pQ-h@Mx)A)a92 zFqv^mOV2`07HA1@blu8hMBVA~`?G#vUi#26&&2;~AdqVFm$3K|F}C2mb%K%F)lMtP zCfBP(`jVG66jwPAbr=e~Pk*gm@M-uV73#fV0ES&nHL*Jgjg2~_4!GqEV`60mGko=_tdba<+QR)ffC7QeWEy zS&Zyo>K4*kL%D;WT|>m1-kX{86iDqz7$!X_>76BRm&!Mz&NT;RZSEayrZM?^L6~pN z2jn3x(w_2qBt0mnHH+Z$7bz8WO0^x8w_aD&{i;f&`i6ybX8$1I&vy06jogh1YJyQz zr?<3x1l9@~$)vF8YEsJfv@7W5L6>_}TaND|NvSF8=n2B~c9_2ivt@qwTJdkpk?!K=~V;S)UO_VfY0<8o3M0M-(a1mJ2z zoPJ=u+Ts0?1zl8$>FmaC*#^OVdq2&gUkV5-rz%3XaWx`Gv$!ff${CF_+Ow5~v@98L zgVd{6#w&JzRE~1FKtT+tSGNdi!o}Osbr2Hhk}wFD5YSzf$HWii2!X1rwCM2K_9xrx zb-?T>xc{Veo*$EDuSID-;5>w zd*br{0DFodG}QNi>HTCj`x5nR!gbh+*gAoZmbyU?`l7B{7HbrA#a`C=t$k{#;9bA{ z`k?%V?c)q@k(ZxWXfMF`4#!{F`=x*ZidbGJE(8nrDICy+ROZD1`{V@!1_-UQBic9q zSh7^9oV&-dU5r;a;hGZ91M16?7XA6G$#^M)`<}qPm~};6cBPH1yaF7PwXO?V{H*Nh z7@6M7Nw1?0?g{Sj#)sVkP@b{Fzns&bHG4m~O{2>Z!kTb+WQ)>NCL>3Ig}rhqP!7hf zX-`$-`jHZ^)lw^&w8JHi0TeUqn&_(t$H~-!YLi+Zq6qQoEz4P>$J{OEP13HqrUVphfZ_p+TWLz)-`$MFKuWU>X&u z8#ATZ9%Pwha%FrR*y4GqU2y+GkPGzQs1-Idhu|1+gCc!M=X?sxE98k%M0suyTVF&? zZv0bkU%vcULDxLCf_*;FU40ru@3g%TPn69gTtB6f%+&xx6Qgx6g@4QmBi1LbY{|D! zjUWE-D`fPuJWtpcAZ=Ud8>AEvw1Pyg(FTumVw+2Fd z-iY$AbG9VPe60+bdoBU<8OCX?0<5QQ5zPWD)oE+18Fwo>L?Ug`@6xlH0h-~vSZQ0A z8AqA}!+c6ez`yE!%lva@X}ao^6k~ftIQ3<}=0P}e;Z@>*Abe!DH~DpJ96Wkc2{-c` zJcGc69H1H!HI-juGgnovVZDYiW1bg|85aW0V|s!U89F&~8h)yie`nlK5(O$#cIIGw zK5n{T39s`P4p9F&TC&;)xRTLB7?@=n^M&bhbC*^KDyAv}WoI`~0P*sg%K8V4oSw4mFm7+V`3j zz}zD_vxLDZ`Tnk#zQMyQTMLBF!u#ox=_?;)&uzXvYd0=R)c!#ZSZJ%RD1iOwCfCk9 zCJquq1a;sq+HKb`qWaJ$fM%u31J%EJV7w#A_EM6 z$Jb5sVo1ATAweW()rD24RRPjN}~VD_`<{Ai=?!*21RcZQzR z2dkO735OA1Zs?I2VY!%$4^ou08UZ%S6WlFkCm;vWHI+vyQuTC8C=M_E{jU#`59@b( z5B=*ye=i=^_vVGoo&xI2%^o32Z)X|jpn|rY9R@jaBS&Y+%lFeyc2UIvXR9M+-SI>{fDp@QuO-=-DYAsuq zl~pdgZZ)Es`@J@^#x`iR=9tl!`FY=J)Z%(b|KT-$*r5T_8*1U8Ax%@K@>Dty4S)we z4$3VsjlwTzM|?n5`NYD+zydOSg?o2(Iiw#8 zw&y5r;AudS1<;6)ugJ)!WJgE8fUR`GfsWG=7;s`U)IF0T0rHwL_t4m=9+Jb@+32^V z*p?p}!$}^$hiQ&X`6B7IF1JQm`~uZY4ix`rZ%-P{&;15pO1@QIe#Q2a6OtNx0k5UF zk;d*ZI#ul?`M3Da=mi*lX%62ReWCZG+zovJY{4Y0Jf$)=_bE$*&#dSDxn)_neL}BAx~N zXrK(5?P6WaxZLnBD`gEpGE?GGQBnWw^7*N>Z)S$e`wP$dY>M7rt+PY)a&rHKkq_ay ziLa+)VP}ivwP8;csV6qGM0IXMvB?H_+wA zTmPm&=M`pnjMFzPH!k=PMVrCJ9a#qk&C9ZIeRS&r;Y5>+LFbbBpZdloI~>k7{dVN@ zB2M6hpD-wAb*u((_q(0}CnsLnX9c>2bp`;*hmm5QV=^FdDfn3Rd2LSG8`fu{VW|Og z&~xEW^qTfNl4d^VG!_50zZ=}YATD@d!U9#?y{5058rMX>MOG<<1%hS;oSC^ZzfADV zpj({a6`iHHS*kfWI8uL7k94v%<$h0@`AHzT~F)C5A+TBIx|=54q4+Cj_g7xNC*yzsH92XE_jR5WpaX*&kHTw&6%bjp05N zuJVBh#RCiC2PuYiW*EG(&vUnmR&|Ktcy8!E-7^6SUjK+hlz4OK7h2EQ#LOZe2n(UW zpe2Ukq=@H>uG0x&qtt*i(k`*R=UK>$4Xqa#{|$}<7LYxHg~i#H{lQ|4Y1>#^6L%pr(AHv z)`$bdsnkcV+D0X)PZj&EgRl8wl1|84y{UIelkbd8a$flbiJ|81zY8^p-gD3j#OUVM zjL8^{haSy3r~UBaIxYmnouGu}$arl(AZ|4Ei<5MGd?TO~JD&6N(~xCUP%cGOH`N`l zVVxt}a{0w0vWRIlT@rQ>JIK#-tF_Mu$X5WFk~nF%V48QAUy!;FeEX2$_RpR?W9dO? zrKoFUrAOP&115Z$QV5W_H6+V*e&Ku&p4empra#E@Cq2)JR6EEOB1mIk!Gtv2kIIgD z%%v+`x?`dkv>$+soM6oJuzGviDawS`=0bu}OX3K-`*LnkNk$^&zBJKF%HoE*&n{6! zlg-~#MMC(;$;tZsV7i&hwKU=d)#QgGxkW3sf**-c+xIIyA6!s4k04>#`h56CZK>z$AB0>b|W{U7P%#Pff-W>6bl=l8+>fBMM^{&K{+AH{;e&*1$Z!9w>7WK#4B z8X(E`N3Vc1VxIr7zW-Y~56lMy-4b!P2nL1(T2fU8T?8CuC!Rk0Q1WkLMwVtm<<;ZNj(!jDACL*2ALZ6RLy)6H13|GM31|MVwlw+6L}>4Y)G9cxgp39&A>hP&po z#(A@@3$b$!=GpH0ei`8zPb=+2AiXbLxPA8x*uEL7Xqx3Aa;5T|n^L?H3&b80KDp^Q|ow<{tXRRiT~p8`=Xb8vmr+pCdW0d@R=pXtcSm z*8=tA2iTNf`+0!r=>GVMGyJc^V-$Y!h*4Cw=&WF(T)(@&}+Xn0RXP z(LZo3wP4icLJ+3$VvBT-K=NbI@eL?s#6f)Of$3K9!Eg^CU{d2ms^p`f z?2<|lYl2f*1VrG#ET~py-~WU8{})h7e3T)g{}{mVesm60{|zX+U(f+5>dyewW9;u- z`qzVf4HgI+IOJjfcpEq{DB7ac^u1+~W}9&0YHn%scfn=qQq+HFnRD|lMYOh=@mDUh z8R-}{IGp*Y>n_(_@8$;!PoCSnMM+##pF-DOxzFzx$NXJ)*O>Ycb>QELx8czeA|5hA z=-V{=wlINQrGot%uLl4}>}#mqpV325%i{fEiKk!g&&6l7t;~S8d;Fl5T5=yH zNKcI^NE2NB<`GFRdA6>Hmz!&LiqOoK9u7eo>KSiE&gv4Io_9O9 ziJp|VTf6^5jy0>bxyn~^0?h?F#uNzuJyDk+*vZIf=bsRwqbD}Ar^C)@s_lkEqDU7? zD*mZlG#TVOWIKQcnLinZi`+bed#eGSpV6?q(5sJwh z=5lgy^_wc{FHlY{u$Tvchww75S(Ucu-F`YtbdhVD{H zQBmoYc@V%nUIU8@v9Z-@FeSaPcq)qK3PqG#{M^=>PM=$>`@~CVbec)0%oX8^ZLL=k zZcZ||o-p6rw@E}5EMokRL{!3_fkB|ij-f?nIBnT5ljF4X(2KoAy3u9}#SzA0nM+gq zm^3Jv2~QGQwiT7iaI6gRi+;Rc9UUfqz->00EDLa)@V;rFU|SWnU9|}z;H%ndRQkpy zzP9UpEtfF=maaCsd7{I64RspMs4_ZFNT-lrT22)!36*OiiLFjuOI0sBam9Y;X+*zP zNhXHD+uT-EK*G@g1wHlM;Wi3mU;jH??S^-Q!@Z+^@k#}5FNFvxlSSS0aaH7W-wmUuK0lX^Jd#B_NqMCGvXD zo9+^aJ&(7F0KInLTotd^S`|;U6(+`&Kd_LdX8)ax?2>v3fxfo#1z1lpVMGdDz(mRv z4|*zB;gNKBemM0FAyoX7llYzhYXf+_Pt zj(X zR}OAm3u&lmR-r7vblaP51$vpeCXe7GGNKEK+9aU6%gmT@NMHY4!oB{6qCR32I3pww3N+)PNqM8MZKIo-jws?#V@Em_{ zcQ{fZv`IU1)Ni!z7!`$JT0v7}WX`^Vr_WqN`SP0BmV#mkVE^86cocgoO3I>c7c_)h zFDVNeN+C4+uy7(zOi|oH1U-T(VWHSq^f)(9vXMdVq5owgy1se3(kILD;tCM&F;9IM z-{^10*oljT5vkAoBAzS+2sc~w>TODNtjyNHWLfmgH0q1LBh2`LU-I(5t2XUiuW>GP zPc-UBF|}H2h5a>-%JAZvx|ec8^onSVbIU2qN1tVawDdqw9jJGBqyuq=LYOHq!?7`@_Ig$?}vM?Q&L6H8tzm^1WINk^x~ zP^dMkA;^kE8qJkCdz7$rmYIOose~%~V8A0>{C+CzFIQsRs3F)33+*vyN*CL#C(eb@ z6Wz{b=}aI$tX*p$b%CO2rYI0pY=o^hya1OvQ(y=YGrb0LTqTuV?E;uq<2H9s2qR(D zxWwb&4YLL_j&g2U=GLa1Ue*^h&T|-m7=d?m6WO3MQfEy3LgouGM{>oV4G?3qBe7nd z{T`aq5dO}?`p0)Q1dH|^>|p5sB<%O z)hpcgROSr%b`Z8M4GB)>5SI(LN|FO4EIGk#gdfmzGIyx4qQg#)FnbJq{+x+=RkBd? zl@_UESDbSUf~BN`zEx#opnGs6V7oUZ*f!k=*mV9EKOH}3>d&k8%rJqiFSh%!baE_=<~y{ws~EUA1|Ze>Hc3?u%=X#CCtN` zg2e-s>S6^tA}PZ}Re5L9DcJ*^Wl|Cs?Nc+ZS}vyOCa^+DPZ5c+(T zVc?Mn;V3!3x^+OG+SYI)hl+vFZHnQ!`Qp|!vQfOWXviH|4Fj0a zWE9;!!hIY=>A0c}J<=0TkqZQlZjGfC(^3l<=xZ1{GfBXnj7hg;%z+b3U@G**G=?(_ z=pA$Dy+l}rEGJttMM*n2PdFNOfmXg^TJ(0)M)(QrXwK#j&PaCY1HX5_LH<+Nyl5H) zOZXE#kN@+K0ww)BB2SuvMMXZKp_!r`o0M%n1o_|4dddHjSg#L<{j>UWjNOKw(kZ?ofWMAjK>_JH96g`D2^_i3UpOTS4Ms@nW~W8>TzHLqL(B&}uV1}aH0Za{ zN9k=wwBxj0fRT0!7Rq<_#ALR8H#aY6m5cxUJThoHNEBp zg{rJ(YD}Qf;Fy=rl8SSpVav+ehdr!xHhq26b2W1ts+XFf1#)0k4C4tH zb!7r*fJjnJb!LNMNsFxN3Kt4o_0zhW*zcq|y$1@;gB2$-M(e+$6_#aM40;Tw&SrX{ zvUq1Eq?8ObaBc5(H0fC5qO-_8bh7gT-&c7!2sYTKjmYAZs zQ9NsGt$xToVnUwEa-X(H>%dhK@`l2o5SknOCA3d=iu4>o6|G`zA!{i(>d;+4MZK0$ z1Hatmc&HOgiB)7@BPRS|w=+ggCRhvdVMg{CRfwS8NrLIM3 zz2?a(-Psymrwsu<**OvZ+CEAR&~oG(0AUQPCW#aG5xp!X5_>c)Jr;|4yr z`0Lt&7f2s5p@ew!X|By3$nq0t`%xjirTa@nY#~=nZPo(Szy`V62KtgN1W3ES49u2N zLWRNsc1u*K%5Gs?3^)wA7r+0M*b;P=5sZt^FJMiZEQ_SNcUSGs|act?V7|^Q)odHAqA>j=OS~SZaAva>6H^@HHOSFpfHxiUP@X%&w!)RkiOeiM7%%p!wCdjV%WpyN@$W;H zc<5LCJndJiHUlQR^-@}F8GW=p6M(F(s{CLb1KW!aS?DC)^u&66s;ucm86FpZ6yInF zb8cAe_&Vg(oUeW_LSV_{Yuun-+4lCfAi+h)SmRee{)n-8;ICP?k|jZ1PM>6YxZ#Rc zdfF!uiN_(qteNBrrYN`utF~M*PB?r{tRv%7>%Hci?Fa;QPLI{ZNaoUgGXO>_(+rbm z6@*DT*Yq0eBtZ}R$$gA~YUo@GF8@V0jH4eMArJc-z#`p(@XNa0*75ZIaOo@vU;mE9 zz4cN=&z}Y6L8s&tzF0!=5lu>W_9sxkIIKyc$db)mrUi0}&QHiFO$EM-?TJK)H?J_N zdcXj8G{N@IX)K={NI+3tG{CQPmQeWELQPL=&p0ZD1a~zGO2nF(tsxr$D~Jn|gRZQ7-})a}y%lDG7b4tiCvwZlN0_u2(x5I~|Q>_A6)?-vbTn zNz%?}MecwpXLsiB#aiS0WhsB<=Y5{OA1G5;7>k_oEf~BsOK9C_LK3l|rZ`60KwDrX z9%1AZ%t(bo2ZhLwd?Xf^c=Mo*e4pc2BD241Y-cnT2Nw*UJN8?SeYBTPs3D#GT8hk& zT5F}3cqL8-d!?H4)McmPnY(I7%?9_p9*4}p-+l+@dHqjUc=gV$g0!r4llbK-+nn8G zBpWs>)2;5()Jn35BYN3tyNVX#C)s?me=Km-XNdILJ%+0Q*V0ugPFohLR$1lNl?tn_ znxypt#!(M-5j0B;UWH3xIdeKV&{Rzql?`e>H@Jy|mtezLZSU@rSz>MNq*sZvR;!YV z%oLfqdich~^JJMuATNCbs-hP3nPka z2d7Z|P|Ym>X9?d6Kb=^Tfoa?^+HWMYtxZ_I9OQv2tRt+wu1(|kD7ArQDsxd`1osme z*emqxq3}S%?A^AJ_+J>Mdw0IzSue@^tl;D}%!3~r#oMyc!9L z*wVqeZDp~uN5!#2|>p34D$ED)02JcXpBdsOPcakgDRE|h3X5b>kn_BccB{*|Z ze|aCsYkKPO2mJl+DU-$@p)p$I{YfVuB0*N{v_uybTy$dUlM2gvIk^tMrclJ%+X+ zwj;hL0hU`R@W|=`#E3lpn{@s<{c(`8Tfzm=iCSvIA@Cu^+#8EZO#XMzo(gq3!!HRF z%pk@mB>G~BB6Tf$erHe=g&oo~18FeUD8tQuUAW2{V z&8iX$bis5#CFr8jBdQwW7)z_)takAMJqi2qT*9TKL?Gr_`52Pw1sNTi^`KX8A%+~s zJaQg1@3Gv?z2-V2c=rt6M$P`PS1W%{*4u)gE%n60yA4wNVTt7K+sNJy=n@wZW}uW4 zkVNx3A?Co%9U%+yruU4g{TVI3loG%3{-;{$fBC;Ano2F%A8z~lrv^ei$q*es$^SP* z5-2qT!2P{mp;c)jERrN#)j|lvgSjNQOjagyF|H{1IUoP*pIh3bT?An0yElq4{l&oN zi4gmCfBFg{NX&XqvZdg$4^XfCZljJ1Ng`B|F|pDJjUKdyw>ww%3^;9%Wq12WB! zsAoP8SC{Ml*_O~W1qEDE@K%P557w?qg$rz18yuY0QzSo>D zB8d7I#Ui1i3b#mPO0G>PEuKCEr5Ex5Oul_t%#0=yBRZG-Rqmbj3J2`=i4Wjup4VhRGJhr$qC+g}&0JEAJxw3MGA{qNj!xlc`97sL zW~7(fAU&Xa@`wd=_vI5ZTho`zXinMTusW8q)Z9()VdLoxxE^K5Qt%w^%^P1C0!S~v z{jXW@qMB3WN2~9y!Cv=!!l@`Oe;Ys^DJ6#CRWAa!uLfj*IdE7b{0!u0Y`Vdh4UOo1 ze4U8n*3~N-_lzM~P{x=~u>V9+J{L;Uvp-b;6v#=tLX=5u=XiiAjZ=5jpUROgj;W#r zVCe)JF-OspM3;a-UNvj6swS~&3|a=bv3P_cR#&t6K-{a2@JLBT2N6e1KBcOEUYm&D zp~o+r9$aT#tVu<-AeOHOY0ul6?$3|pfRFpfRv@k0kZ7+>@{z9KmEPVmJ0LnVu{v?^VJNF{CkvaT;RE7z;7BXAbq-dJq8K-4S z;YJkYygoJ36FC$e4}7`2$SV3RUL8;bB?)styyQ5Zy0a_v3~F<&Fe4~fX4E+kaGomu zvXW&>LxXiAM^$r?F6q*|DHp?5$}^6XvhxB5vmW1}SMhhw{G^Grfje%awq<(g28F{P zzfe8et8&0zY(~Z{ye~*T0T3J80aidPRhSZwclo3=_qfgk{_JJ5P&t)| z9v5q#=V=n=B@r!ICy`-TFebR)`}eVqCXCVojEkOYNJkQnmKpVmY+5yUK!-s`^GjWoEfqVrzz$GK1%e z-DcH@RAbsLOl7NdU*#%&UBi~jV;`}*cB%~i9G>fFOK41Rt^%iK3ywg@i5YY)^=gnJ zxbBHc9i}x}XDmC)Hd0$rS!)OUNWRY9eF#7XL94Gb&|%q=(DAZsSl!G3OB^s-^jmm4_te6(0uw4zR~ie zUUUTM)%ai#cI$eJ%~w&uwMsRolERPB{Mo0<=%LuAi%P()N>?h%;EyY+7M5(Mnr#3$ zFGH|eKhcxbQ`dNl4wmrBRgmmwJ!zQfWak|v7-;ZmAfQHi`9J@|l-o00Ao(neYlK&*oU{BN3>@1X|*q8RcSw6qAT6Upp@zwB7@0rr3 zDj{B_pUP05jFpQYnnLlnj9O;!{p|qw??=s4bQ=0rBuF7*kF>>^Dp9=!KI^OcihRM= zQQ%g}IA=+S2kjP5)!d zVvX>wod#>901h8qhOFMTDW>PvU1j+?3;Nx|g>y0@gI(bAEfF zjHDv(!qA5fweWjCiAWvUg!1!+y0D9bCbodg3$FN60+}bU!Quj{e{PsGzCcz2DPo$^ z;MFir0=F8L7}$t$e2_pTa)nvGGXSR^5i|T#gJgSfDY&u!INI8Q{$GxU@@|gC3d;51 zda4)YGNejCH=6@6mA2LBv@dDOf99~RA9W%4X%`jY-n6z1-BEz>*I^cw?5==kJg z_J=i3+R(6~YM|?&#TD_M#+jy~$hN-+W+InhW>1`!Ba-c&h)ZM~A8dv1sV1yf9n4CS zUF9jB@EADksShX*s4b_*I;gZ{E?a1$1alsFT#_?Wq~?gi){Ne-<5?2`xmn}up0i4n zh{6zNuECHw8kFAUE=-F5g9~&~^Zc}Ag5Wyk+G;SiT%c1l2{36NB6yyj#lZ$rpIn{2 zgd(Sq>Q2Hjh6`B~EZ|U6T*pt`7kCzFVUsD9s+<&B^Mk{3&OEYQFl{1DLPpJ6=?4z}w?GinK>U z-BDHn^z!dXM2F|qNqWD4G(_FCWbNrh?jQUm7+0B| zK@C9Mr~|cjRE`ISMT*Tipj?@vRAksq?NM|_iFD`cK~b@2h1dnGi^@>15lI-w$li-c z5F^M#uhQm%*#1)EI5gSBK#Pg){$pWM1sf?oe5p5gQ%>j}AB|G={IVTEF#~Tn`0r1` zmHMooWB7N4!>rs+*JL+~u+$Y1{1@k7UnB9Wz<*9Zvcc|x;vcy6@G}AVet5$F>atAw zWa0VY6c!ZFM!wx~x0cbdX~N@VMAtwY;1h_;pr^yIC^HS1`qVL(Co``w?0E}rW9%Om zUW8FX1~L7B1d?wzTPA^}Atn;9vNxRlj_WV;dV2go(}xmc!JK#!M=7Ofs9d!qU^ti7 zq7x$>llRk2XS3;NoJ8?ju|oA*};!&P&54q1!f)U~GImAm(FIdZaV zxU)9(_UDznJh1M!wUVz!ohMxB=b+*z8j#hRXL+M)a#en}j|teN7fCU;NhmfnyZZpL zFl`xdhE(MfH-M>U_Gdol589E?ur6H~i_+kNKp|4-5;f8NypQh>jo}kziUadifXIKN!I5~Bj zpWySKmiB8UFv{b(9@;KF=+!0Ee4#Ro<|ZeXDMimX{8CmpWR~G&&~sxr7V{%Gts&UiyJ|oB zJS7@Y&ddKpiK&1||4O=IS0XBM+-oCC=Fk}@FQS&+l3o|2?sA|;7emh{0vDsP#~n#B z;GhQ#c5$JMmd`TLxUZ;qMVF5n>x9HYgD26T$n)6k z$x^vugVgD&1$$+Q$8aV59b7VA5GRd}#{3f?2RVa5!_s z$SSC#Myf5Om(IB><3v|k_N&Dzk-XCvZa|^?A+qNv1fumkq=AyIqnqt3n|M-8mtqsW zD3$9e)R+*7ESAi>2UFK!SYqL#G^YMB-Tob2X+lmK>nM{v#%1Nyvb>BqA~|#+ePI?5 zr;KA~q|w(Is;!|=S7r!30p}c$QPp`PQ*Q>nLfcy5mu^^>rd_dRmd-7Bsa+!IyalVh5uf;&Fi~wUZ(l6m z-DHwknVOoKr6=GY?n*}Oc+|hTcC-dGZWp^b)z~mGB<9Z)ixbWxK^*#kmM3R>+yx`& z6XO64U^n~@hFZv+r*!Lcc|s@lb;8`0|I(b*jBKP+;Rqy~FbwyxjGV*PngPAcS%gA9 z28Vl}j53i{QOSD=91RocrW)$I!{?h5$)#f0Lm9$mAl<*e_){B0bC!opQSl3)wZI(2}Zau4vj_+YRzq) zluM#g&XVlJlLMt7J(d>v^?Qz4ES#&FmCUJ7H){3&p}Nw^|4nxM9I+KYM=a<6IbzM7 z3{7m3Y|)Yb`*usnjRjEo?*rHGYI#nP2d@^U+aGQVCWu84DH4idj!v`JBy=#&l&!JI zT51Cs5GoMN=m$(1vD^4PT&5p$+yXDhLCv5$GvPb!e8qqK;P?6X3hOVHOm4zNnxEJ6 z#zbo*7YL-<5AvG?LyS&J(wR4gl0=QmM|Dsc*_vb-+lxa6u<^i&od|9kCq%^y2{BMz zVTsFIebl|%J@-t12d9?PZg+WFGj=&j{cocnUP62s_?=hXr6>cY+`jyh9aM`l$`;V>InWff;f39@tRFTG|-t?rsP&$%95E%NGlL>rc zaiR$sK$+EC(ua&ZfY|HVf$}}naQ+aS{GF%}pP8rpNQY2ro#804}PpqTucq|f{f88*j19PhW{j%g^34=`k zv&x5WlgLKgA=}}Nb!nHP7LJ*bu?G8qehPqj)jG%|HS0Ig{5l{>WKWDJ`&H%bD=yM| zD5so0OsXZn=jcCEh$IQ#DkY~l#X_aD$aVTKQ8K1H$hT&hIDz#I|DUMhz)U2)^rz~& z@TX>v>wifcM1Dz77+<_bEq)K+|DV01VE9?AGHLghAmCf>N4)s;A@Fnw<8z zpPIb-e7pao^#AdtiV4W%3^v8GMGDgwGZpUR(7`wIu%U%e!7``y+wSt#?$(5Q!$A`) zV>L`{06s`t`L0%~nz>v)*Fi+&XT!I8OQ#CXJ8pk=Y4=v`mXe0K0GOUZG#Ou$r^D?z)kJ@%5vL=0B6YDOKj$QkU8dw=9e;b^|HH8+&C8vTLlOufxa zGmTK!^k;?SZDv>k>XF>AGVp08s(^E0=!ifq;D+b$Z#tt3O%s6r=qXISllau4Z6-c4 z=%_vv5{_|3#ogV{bPe}#p$f^A_z26%Ji_>7zvWU?v4&4TdN<{RyJN22!Iyp&+V=P_ zZ79=bC=;nLoQ~kIAV!cd9~+ZD95Z7I9GyXe;Q?T?l($CRRTwm`&g37@FcucKeqVYh zU<9t%`POXoV?cjv)9vCV`E=B}z#H+j=2-0gk?P10ugiI#J~SU8iN3Rxb_N%Iq!wjM z4#BoWZEA7;)=rd3ZiZ1-UOnGUiP16*oxX&f+BWD<)bDi&OK2^~NfENiQ{OT(M?xF< zP=qlibb|-4^gz8-!OU_jigyoIIMOyaz-W%-z=0f@l7;+&MR7zC0$t|LAG0LLlh?p{ z!XFiM0*V#m8a6%TY>&~y$)D@RK87JmvQmP8A-1sri@g>Po#Gw{)4OVMLJEp2u_%AB zN(Y%tNG7+qN^25q%J@XtqUf>@aaxa*y9$uCYz6}1HD3JK9hOa-c05ke0ntigAsoB$ z&~ROZBuNvcwvciTGnG?2i4#&YR1_^Cy>SrnLm#xC5?q}?Io5oAibu@<-16o^jt$94 zn?k<99o=>ACST(C;LQNE$j3kMgNTDiYCHcUSEUT&GHg=WzQrkP0 zm8RDzSP(H+2}ok@UaO(PsJ@-`6gbLy?tXSP>+$jaM(!n1b9pcp4`aZD_x#zvDwAV} z389{UWtSK0{^SqF>$K|f{;0$5LUbXT_bvh2a3bs*&ABbHPjugZSDmR?b?$Nc*U-D^ z_qobLA>Df8*SHaEXR5=xFF9zp=jw07PF~@}mt5(4_QHwkpjy@LKT6gfhPsV2;72&1 zy~#Fx*Df+{KqqE&^~)#!uor}9P9Io@?TMzxI%JtaYFfvUe6b}D!*&J>3a-qH=LSe)H^&qNBskm1IX#DJtphcl8nW` z)Ry4sIO{~-2hX~ZQTdS?_KN}(O|Ss-J@;lBpEIy7#TpWHp$X@UM(Jmt^0VZQbgkF~ zqBEg(>n63V8h_kNgAD!k%3&BniY?+vlITdY(z}2F5(pzm)lwM;F!Kkv_&|0B!E{M? zgJA+9`FwS%d|?IM0VUlb6*iNfIMroq$|r?q*eIPj{7F~TC|2*Fi;$ zR1mOZKd*rHB}a`0>+HX=$c>K7iG{n=g&j?SA}cd$7PvOB%`jWXaJanbCjL#S?=v?D z`3>|)u`xk}K5m5L%)`iVL^vqkdsm9l zOKOxYa|EmrH(ICPY9{Gw_toTI1)|YE=Mum?Y-qm##`wZ5o27WEo+MQ#CXI`#017f? z)OC@I>I$ae$=!q-qk^${)!?XmPX0CbmH`X0QxPwj>!XTim_lYaBWM~aI=*jory$BQ zZWrsV6jyh=aVa6Sd@S!2s_-@}BN2A~u+|}5Y#qudT2=@}e)fyHOWvd#vNGYEsf1pt2Ty~*hRjLVP0M9qz&JC&m|uG^roRGmxkYE@cQ0v4V!dqpm}<@ zg-1ot_8AF8<9Lr6!drvE|JT-4fJM1AVOT){32By;Sfmk`kQTUr(jna3PRIkzN75w#pa!^;HBKjUt(o{nb z-wop8MI{9<^5m3Tvkea?Lrt4Y*k|NevjWSrS$CALr&sXhm=2P}t9g{idz8ER20Q5R zb&0nrRwwtMl5GnU__b^n`DZ9N@wkzH5{K*uweb))WdpHesf?}**v+nTR}{&ngGhc8 zdv0WjULymlH`}lxcnnAZ5)k3f3qMnJtc>+LqtE1$5(PYfYNFHSqQ*A!S2p5!s!NgT zVrXD_{J@zAvb^@QojnN~c8{WPXDf4N??3M#4&z03Zq`;mu)oXc2#EG6svZ2fLW2+B5bDIUi|nr&Pz z)~H~d3##E%&sDL6%a>Hto8na%N7<5+->c7;l3QJP(C_v72nK55c-TU2Rjw@y#q1}d zM&4g{!{{qC2bc7!wQ#6EU!gWwH+#*s(#O`qt3sR>%)4H$J8_H@ zsjAgPF#D;}g(Q@j%^Qo@bzv@Qx(fC2oH(I{`_k zO0m$cOdLKjD4f&HsBl{; zux*^DVs2KWi85F>_%a{f z*rI>Yk~ZBM-zqb8oNX`dskI!O+;JvP;3nhe&<#lj4FlgZhN zIx*(R_7O&;{U!VSWv*9HeRV~CVU2tQhDDqWrOi{TYK*WVK1F=UQVs5DwjkG|=MUJQ zRVDPkn`TJRdnuza{6#I4;HhxC+SG2L@nMvV5)}z-C1wH2SQKtcs~{S1Gbx-GsX~4L zVR~ni`M#IRQ9N^!;dI`m}>MO$OjBT!)8KlkZfVA2R2pl9u z|E2KV(Bw!uF-mYSNmGAhJ1|sQ3op^W@#Zsa*xgjtg~BD^6ftK}L^hh71bS~re&%W& zvk|YC*{kFTP`bzeX%JW7)D@Wq-EW$2%k7m;!r+^hY1|HnQ`ldZ8^z!ZgSVT9k)ql0 zY=P7aGn<9BoH6v8iV%_{A9ccZ#}Xg!v_eCY^VQmGiW-U(X@&wNs5q-M$DY})`wut5 z{H|eev8bo%Y}!TZQs<}|3BOiP%)h3mJK*N^Ate!t`Cc9_np8gng(BaL$9*=nqCsMp(P#)(Z1Y+uhfGMnM#v zM{nTBLkRhX!VST0znxn?8Wre#oh^8lrwR3t%0P0esBqSKzKs(J-;AlHZ^-VBl1JW= z;)-X-qJe}hzQ*rtE4U-oOs~b3`IOQzR$tMra$>w(x6`&ZNFyAoRrk6e+#&s{KCE}$ zMw){x0_+%}=5hMD5-UhgqYG=RUb7gwr03W~FTf%@%c4c$i13<;N=81BvTNOf&$88k z_h=w&-)66NmebetyXt7yz0D4dAip!@Vlcq#UhMxqSceQx=#vd=}z*E5f-H zT0Upv0(zk%s4zeMfiFPz2FDRsU7anyQ_q4Nh zcGa1tGJBGK7t_SBg4`%86s0i7o^A?oeN@nBW@VaeC}Qg2CF8(iNR8V)U<|WXc3|K6 z@O)bLb>(rH;}=Hw>0hrUx5|x~h!jb#>FI@KSv5-R$-IPe@+*oEhC@PK;Y2i@P&SK) z;A67l$)#Ml2-^#*J!mYxYHN;(97m)~n`kHJ$bANiP?eG5NaMie4oS{VP8yLiGPrii zZvJ6;eG#;v`Ht~c`{a^?)FF&rnmNwK(Ch1RasPT-jAv)gSf1lXT)CQ5TGXwzF?!Pw z$k9vs5V)7vFqK+Ih~a2^i0hGSweiFxEEV@!pfXA(9V)^>v;UaRPRv4l7L(Ly#r+O- z>9E&+e0su1$N|nZ-&T)I?G-VGETzXho0HwN0j}y$hwYG_0G)vAQ zw0N|&vvQa%t5+~1d;c#N?avq*uL?H08^n|A%DpMf`rZ*#WY8t&>NSw!PxzZDu^_)V zfR`b|lA`fK{<|_xT5TuAtsecvWj^7YtS;=j*Pmf{R_Djtb3}W^kjB==1W=88o-J$} z-%XQ_62T@KBXE3YWY-MqnyGm$m1=6(=&3gwGM@R;W0L4uAkGReF@JpmDqcj-pv5}e znO|M0y7}nY~tdBcPG$iNfeKviaVVplR;x*UwC3S|yr`Y4! ziN7YCNI>5#{S{Ug1&krd<20_eK&*UDp)?JwVpLTR=sHB)$0h&PnLarpcn#yzvx;uX zrRXbRcoIn5l28Fy{oGS;+fvKZS)Zowcx~MoG&F}=fqubGC)fn&J!?jk%=M7%dmQdF zZF9SkOP$e3k&nLZxQZ*&B0JV=N#Kg&&)vimMdi2cWYu)X({?39F!gvgRE2L^g5{f- z+j7BHJW#L46#`QVUksQ8n2-^2h@7)&i5^S7#f*gMBo9$oKE}?n|3uqrWm=0-^n0%K zFAi)%Wn{kOxAF_(v#1Yb;GdbTPi@b3>ue&PYIwI|9F#Ba5N6L)C2hHWwI*~FTyUI5 zck4Tuw-rPC1nqHeTWAa@8lp|OLa@A`6p5Qm>pE|DsI^=hYpC;8!mY^?F)L$0-{pf4AfCy>UyIBFtM`E3BC()C$1Wk3DR!jTz zM)qi3=;4;D-95|YID)>a5lC;A&)3_CQTry9#x;$_GC}?=Z3((h&EC$0l{FyCd}~Vc z$QPewy`o!8Cfx|^@^+-lz0!2oNhB72I*IphX28VkzlAS4fUwI{+HrS(6C z;fyO6!Va#THs#GmwE2o#acQ(FKB+Gl(L-p(baFkB!9_MZjYwzQ z%{FaQ_-sLpBZuo{adAa6!TaD*dBU}c=s}xVR~vIPZl#i`r_#?ub8o4(9*QU!6Ddd& zub1MpJBZC3U_H80y$ITW1=f1R_}-Y<)D}wHMFtazxV}3%6>7A|&?w(`REn3Lh1osA z#%SE)z&F9g{WmTbG{e> zP1=Ipr3tsNM(MO`iG9)^b*h>}A1kFr2Z#*}wZg5N!oPje*f)uMRW^E+KD@;FcAgA9 zOkJ9LXx8;KpxjS8txX;Deg4+6;zIu5W25%kyDwH6{Z^WzEpql#6$fu+yb~P!ay%)I zyU!x3u3Znl#>sI2B|brMa4Mx`h`t{NyTH}w1&37jtyafSDe#qPoVi`h92(V%Zv>_c ziTLxFGmyGrd*Qay(*ioD%mlo1~buh55B4xd*6NDhd6`29F8tGjG(reBt3cPN&7a& z_Y+>qi9K1+R{N44GPAY6`{?RvkOf_b;A{}v7<(Bv=uHGGuw}NB z#BeN^_Chs&_AM_qN{%h&{OGJy%_WVun%+0?_zb&(A6ZckjK)6FDQbKd%tzKx^|hBY zXvICP`@^z+)jKB3^-dv5-Vz3@;jg@}pIg>7+XvXj2m~}vFW#>NzxXt8xAAkxLtMkY z9<|k&Za3`P@8!Xeic&i3gmj5zi$K^~6$0!i;I!n^HAmky{S<8UclZhlJ*~StMR34a z5S|_uX>>#0CGRWkj-V*hsQ%;`2 zkMlcsE&5lLdyhRm)XgUGDBP*5l%6%cHYtuZZZXmIb87^he8$A%n>e-&Tg4(Hk|J1q z%&@GgP>FI@*me-=P42{@HXDSQLO7tVfkE9%2Hq4-Bp=q-iObc~yt_Vj`J)Fej8~ zM}>OY-G?7l94uAsB6542`U0kORF@JpHmdH;ZNw=1`PkTUHTFCZefnMkgTR<7><%V3 zXYawp=D{r!wx?AGncgp?-ov65or;28J0rg5S!s^%p$Ag3aZJg#2r+Bq-0Lu1L+0Oq zH0-I#7h~<-0+$v=(RVE{eq^SrQ2AR_^a$JfmGTH&e8I68xSyS;U(6yBCb9JOrMu@{ z7aY?OlQ}lS;>gmy`*Uo;YBhcIE1Gq&7WsRe?!lJbpKG`x8S~72mBe7-`9^O&Nu)J- zp-j1C$Gbbn1S{sn<|X(_b$5?O3Ts+ZDut)}ZB~XJpDCZlUw2YEY;Q~b+G5}uXRS+QofUJwn+3c zP8;=}C2yAOEukuf9q`sl%wRNYn#8PzyU6eV@KP#h4REo5G_uAW;F4G~oGU=SnX={k zANgiJC~$?ylR9*!-~WXAJ|&(>3ItS%f!`l_Zs$Q&Gfq<*D+Fqp6v%bMCPe<9TsKmD z$Yrp^3KjMhAQcWBL~;cOfsRXxP)E|E<3w|T5s^q1!1oW)0FwIWr!yqu&rbr&>=&wX z#6sTZ)B!;+Fc9?OMz}9CA%^xyc$s*(Ol?e@oPL`Bm*;GS&hds06AkUh^8nX5{QJuG zW4wYe9wh~oZvt+1_837CSm%Nl9yd7twDS52!vkS}DLn;L0+IOF1{zwnpFQ?ZC2WMJ zC*(hhBlSv~96(3k0zqlsONu!7|0-VdrTC8`6I!p&A^{qj_DwW2`b&z>34bbL;az%U zgz-KjsDS=gBjRZS8Nk~3XRup>F&8p7Q_S)s7ufb7sLGV6^)4_?j7=7uzKlcj!RaP zB>}6{e~d`SAtT63>%7BfheQ`*q5qV+qdOu6vFM&R)Ar*bw2x>nxVltv?UJkO`ae~O z{~dzX&MATvCX(M%tnmC?R&Fc!l(8=VaMAu*)7-$C=KeKjUac^KI$RJ@xKPCH zy&pNJe**$#H$VVJh_12$0?SoK5Slk)o`o8bze)oL`$xXtYKIAa|+3h_E#rkO&+QG9v2N@GcD8k91afV9~Sy^+|+(bsl)n_PZ5)4!^N} z^FsFPd48VQIvJ=x^k*w3qJK1B z3*G$9Xh%a6y#(!yIk#WC!EgcoCk61AMz$X+`5aCOr9y~pUcI35gJx9#P+B^G-Q%){ zoiqgcHY)u0B#Lz$l{ma_e##;JKIbb;J@Q2Kmx)gxJNkvXF%xf-~zLf*5Kk|8S6mI z?=T=r&YK!{7(w#`fPfdkDO-XO+$bCn<~xKiN)izGpcp~ZBM6}y^*>3m+I#>2 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index e411586a..df97d72b 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 4f906e0c..1aa94a42 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,67 +17,99 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -87,9 +119,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -98,88 +130,120 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat index ac1b06f9..6689b85b 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ @rem limitations under the License. @rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +25,8 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,7 +41,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute +if %ERRORLEVEL% equ 0 goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -75,13 +76,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal From bf218f54c64652d9b97a52fb35f84a6bfe16520e Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 9 Nov 2024 22:56:57 +0900 Subject: [PATCH 2/3] Use pre-commit to format the source code --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 2cddce91..fce56e27 100644 --- a/README.md +++ b/README.md @@ -621,3 +621,17 @@ Fluency fluency = builder.build(); ##### Other configurations Some of other usages are same as ingestion to Fluentd. See `Ingestion to Fluentd > Usage` above. + +## Development + +### Pre-commit hook + +This project uses [pre-commit](https://pre-commit.com/) to automate code format and so on as much as possible. If you're interested in the development of this project, please [install pre-commit](https://pre-commit.com/#installation) and the git hook script as follows. + +``` +$ ls -a .pre-commit-config.yaml +.pre-commit-config.yaml +$ pre-commit install +``` + +The code formatter is automatically executed when committing files. A commit will fail and be formatted by the formatter when any invalid code format is detected. Try to commit the change again. From 70cab0e8759fbc15ee8400a7d7ec078e75940caa Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Sat, 9 Nov 2024 23:22:25 +0900 Subject: [PATCH 3/3] Fix some issues --- .pre-commit-config.yaml | 8 ++++++++ build.gradle.kts | 4 ++-- .../src/main/java/org/komamitsu/fluency/EventTime.java | 4 +--- .../main/java/org/komamitsu/fluency/FluencyBuilder.java | 4 +--- .../main/java/org/komamitsu/fluency/flusher/Flusher.java | 8 ++------ .../komamitsu/fluency/fluentd/SSLTestSocketFactories.java | 8 ++------ .../fluentd/ingester/sender/SSLSocketBuilderTest.java | 8 ++------ .../ingester/sender/heartbeat/SSLHeartbeaterTest.java | 8 ++------ 8 files changed, 20 insertions(+), 32 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..db8125b1 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,8 @@ +repos: +- repo: https://github.com/jguttman94/pre-commit-gradle + rev: efaf1704425cc50df91d63920e8353d9d081a622 + hooks: + - id: gradle-task + args: [-w, -o, spotlessApply] + verbose: true + diff --git a/build.gradle.kts b/build.gradle.kts index 3c9b0879..4a833312 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -14,7 +14,7 @@ plugins { `maven-publish` id("com.github.kt3k.coveralls") version "2.12.2" id("com.github.johnrengelman.shadow") version "8.1.1" - id("com.diffplug.spotless") version "6.25.0" + id("com.diffplug.spotless") version "6.13.0" } subprojects { @@ -195,7 +195,7 @@ subprojects { target("src/*/java/**/*.java") importOrder() removeUnusedImports() - googleJavaFormat() + googleJavaFormat("1.7") } } } diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/EventTime.java b/fluency-core/src/main/java/org/komamitsu/fluency/EventTime.java index 43195026..5a7da282 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/EventTime.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/EventTime.java @@ -85,9 +85,7 @@ public long getNanoseconds() { return nanoseconds; } - /** - * @deprecated As of release 1.9, replaced by {@link #getNanoseconds()} - */ + /** @deprecated As of release 1.9, replaced by {@link #getNanoseconds()} */ @Deprecated public long getNanoSeconds() { return nanoseconds; diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/FluencyBuilder.java b/fluency-core/src/main/java/org/komamitsu/fluency/FluencyBuilder.java index f3a65fe6..97153381 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/FluencyBuilder.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/FluencyBuilder.java @@ -69,9 +69,7 @@ public void setBufferChunkRetentionTimeMillis(Integer bufferChunkRetentionTimeMi this.bufferChunkRetentionTimeMillis = bufferChunkRetentionTimeMillis; } - /** - * @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} - */ + /** @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} */ @Deprecated public Integer getFlushIntervalMillis() { return flushAttemptIntervalMillis; diff --git a/fluency-core/src/main/java/org/komamitsu/fluency/flusher/Flusher.java b/fluency-core/src/main/java/org/komamitsu/fluency/flusher/Flusher.java index dc6e76c4..91047e76 100644 --- a/fluency-core/src/main/java/org/komamitsu/fluency/flusher/Flusher.java +++ b/fluency-core/src/main/java/org/komamitsu/fluency/flusher/Flusher.java @@ -173,9 +173,7 @@ public Ingester getIngester() { return ingester; } - /** - * @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} - */ + /** @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} */ @Deprecated public int getFlushIntervalMillis() { return config.getFlushAttemptIntervalMillis(); @@ -218,9 +216,7 @@ public static class Config implements Validatable { @Min(1) private int waitUntilTerminated = 60; - /** - * @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} - */ + /** @deprecated As of release 2.4.0, replaced by {@link #getFlushAttemptIntervalMillis()} */ @Deprecated public int getFlushIntervalMillis() { return flushAttemptIntervalMillis; diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/SSLTestSocketFactories.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/SSLTestSocketFactories.java index 54a8504f..47c04d76 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/SSLTestSocketFactories.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/SSLTestSocketFactories.java @@ -55,12 +55,8 @@ private static SSLSocketFactory createClientSocketFactory() { } public static SSLServerSocket createServerSocket() - throws IOException, - CertificateException, - NoSuchAlgorithmException, - UnrecoverableKeyException, - KeyStoreException, - KeyManagementException { + throws IOException, CertificateException, NoSuchAlgorithmException, UnrecoverableKeyException, + KeyStoreException, KeyManagementException { String trustStorePath = SSLSocketBuilder.class.getClassLoader().getResource("truststore.jks").getFile(); System.getProperties().setProperty("javax.net.ssl.trustStore", trustStorePath); diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilderTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilderTest.java index cce42d84..0b744e01 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilderTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/SSLSocketBuilderTest.java @@ -47,12 +47,8 @@ class SSLSocketBuilderTest { @BeforeEach void setUp() - throws IOException, - CertificateException, - NoSuchAlgorithmException, - UnrecoverableKeyException, - KeyStoreException, - KeyManagementException { + throws IOException, CertificateException, NoSuchAlgorithmException, UnrecoverableKeyException, + KeyStoreException, KeyManagementException { serverSocket = SSLTestSocketFactories.createServerSocket(); } diff --git a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeaterTest.java b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeaterTest.java index 2d51dfea..30080194 100644 --- a/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeaterTest.java +++ b/fluency-fluentd/src/test/java/org/komamitsu/fluency/fluentd/ingester/sender/heartbeat/SSLHeartbeaterTest.java @@ -42,12 +42,8 @@ public class SSLHeartbeaterTest { @BeforeEach void setUp() - throws CertificateException, - UnrecoverableKeyException, - NoSuchAlgorithmException, - IOException, - KeyManagementException, - KeyStoreException { + throws CertificateException, UnrecoverableKeyException, NoSuchAlgorithmException, IOException, + KeyManagementException, KeyStoreException { sslServerSocket = SSLTestSocketFactories.createServerSocket(); SSLHeartbeater.Config config = new SSLHeartbeater.Config();