From bfcb9b6a2f4c0b316593714bb7ce7ff329190a0e Mon Sep 17 00:00:00 2001 From: losalex Date: Tue, 10 Jan 2023 12:44:18 -0800 Subject: [PATCH 1/8] feat: Add support for batching configuration --- .readme-partials.yaml | 19 +++- .../logback/LogbackBatchingSettings.java | 87 +++++++++++++++++++ .../logging/logback/LoggingAppender.java | 32 ++++++- .../logback/LoggingAppenderLogbackTest.java | 57 ++++++++++++ .../google/cloud/logging/logback/logback.xml | 57 ++++++++++++ 5 files changed, 249 insertions(+), 3 deletions(-) create mode 100644 src/main/java/com/google/cloud/logging/logback/LogbackBatchingSettings.java create mode 100644 src/test/java/com/google/cloud/logging/logback/LoggingAppenderLogbackTest.java create mode 100644 src/test/java/com/google/cloud/logging/logback/logback.xml diff --git a/.readme-partials.yaml b/.readme-partials.yaml index 3a9925d33..4135679b4 100644 --- a/.readme-partials.yaml +++ b/.readme-partials.yaml @@ -17,7 +17,7 @@ custom_content: | application.log - WARNING + WARN SYNC @@ -43,7 +43,22 @@ custom_content: | - true + true + + + + 100 + 1000 + 500 + 10000 + 100000 + Ignore + + diff --git a/src/main/java/com/google/cloud/logging/logback/LogbackBatchingSettings.java b/src/main/java/com/google/cloud/logging/logback/LogbackBatchingSettings.java new file mode 100644 index 000000000..48e8b7b9a --- /dev/null +++ b/src/main/java/com/google/cloud/logging/logback/LogbackBatchingSettings.java @@ -0,0 +1,87 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.logging.logback; + +import com.google.api.gax.batching.BatchingSettings; +import com.google.api.gax.batching.FlowControlSettings; +import com.google.api.gax.batching.FlowController.LimitExceededBehavior; +import org.threeten.bp.Duration; + +/** + * This class is used only to provide batch settings configuration in logback.xml since {@link + * com.google.api.gax.batching.BatchingSettings} cannot be used as is with logback configuration + * described in https://logback.qos.ch/manual/configuration.html. All data members below are simply + * copy of {@link com.google.api.gax.batching.BatchingSettings} class, so it could be used with + * logback.xml. + */ +public class LogbackBatchingSettings { + private Long elementCountThreshold = null; + private Long requestByteThreshold = null; + private Long delayThreshold = null; + private Long maxOutstandingElementCount = null; + private Long maxOutstandingRequestBytes = null; + private LimitExceededBehavior limitExceededBehavior = null; + + public void setElementCountThreshold(Long value) { + elementCountThreshold = value; + } + + public void setRequestByteThreshold(Long value) { + requestByteThreshold = value; + } + + public void setDelayThreshold(Long value) { + delayThreshold = value; + } + + public void setMaxOutstandingElementCount(Long value) { + maxOutstandingElementCount = value; + } + + public void setMaxOutstandingRequestBytes(Long value) { + maxOutstandingRequestBytes = value; + } + + public void setLimitExceededBehavior(LimitExceededBehavior value) { + limitExceededBehavior = value; + } + + public BatchingSettings build() { + BatchingSettings.Builder settings = BatchingSettings.newBuilder(); + if (elementCountThreshold != null) { + settings.setElementCountThreshold(elementCountThreshold); + } + if (requestByteThreshold != null) { + settings.setRequestByteThreshold(requestByteThreshold); + } + if (delayThreshold != null) { + settings.setDelayThreshold(Duration.ofMillis(delayThreshold)); + } + if (maxOutstandingElementCount != null + || maxOutstandingRequestBytes != null + || limitExceededBehavior != null) { + FlowControlSettings.Builder flowControlSettings = FlowControlSettings.newBuilder(); + flowControlSettings.setMaxOutstandingElementCount(maxOutstandingElementCount); + flowControlSettings.setMaxOutstandingRequestBytes(maxOutstandingRequestBytes); + if (limitExceededBehavior != null) { + flowControlSettings.setLimitExceededBehavior(limitExceededBehavior); + } + settings.setFlowControlSettings(flowControlSettings.build()); + } + return settings.build(); + } +} diff --git a/src/main/java/com/google/cloud/logging/logback/LoggingAppender.java b/src/main/java/com/google/cloud/logging/logback/LoggingAppender.java index f02a3bd29..19614454c 100644 --- a/src/main/java/com/google/cloud/logging/logback/LoggingAppender.java +++ b/src/main/java/com/google/cloud/logging/logback/LoggingAppender.java @@ -63,7 +63,7 @@ * <log>application.log</log> * * <!-- Optional: defaults to {@code "ERROR"} --> - * <flushLevel>WARNING</flushLevel> + * <flushLevel>WARN</flushLevel> * * <!-- Optional: defaults to {@code ASYNC} --> * <writeSynchronicity>SYNC</writeSynchronicity> @@ -91,6 +91,20 @@ * * <!-- Optional: specifies if a batch's valid entries should be written even if some other entry failed due to an error. Defaults to {@code true} --> * <partialSuccess>true</partialSuccess> + * + * <!-- Optional: In the asynchronous mode the call(s) to Logging API takes place asynchronously and few calls to `write()` + * method may be batched together to compose a single call to Logging API. In order to control the batching settings, + * the `logbackBatchingSettings` section can be used as shown below. + * See [BatchingSettings](https://cloud.google.com/java/docs/reference/gax/latest/com.google.api.gax.batching.BatchingSettings) + * for more info regarding parameters shown below --> + * <logbackBatchingSettings> + * <elementCountThreshold>100</elementCountThreshold> + * <requestByteThreshold>1000</requestByteThreshold> + * <delayThreshold>500</delayThreshold> + * <maxOutstandingElementCount>10000</maxOutstandingElementCount> + * <maxOutstandingRequestBytes>100000</maxOutstandingRequestBytes> + * <limitExceededBehavior>Ignore</limitExceededBehavior> + * </logbackBatchingSettings> * </appender> * */ @@ -131,6 +145,7 @@ public class LoggingAppender extends UnsynchronizedAppenderBase { private Synchronicity writeSyncFlag = Synchronicity.ASYNC; private final Set enhancerClassNames = new HashSet<>(); private final Set loggingEventEnhancerClassNames = new HashSet<>(); + private LogbackBatchingSettings logbackBatchingSettings = null; /** * Sets a threshold for log severity level to flush all log entries that were batched so far. @@ -224,6 +239,19 @@ public void setRedirectToStdout(boolean flag) { redirectToStdout = flag; } + /** + * Sets the {@link LogbackBatchingSettings} to be used for the asynchronous mode call(s) to + * Logging API + * + *

Default to {@code null}. + * + * @param batchingSettings the {@link LogbackBatchingSettings} to be used for asynchronous mode + * call(s) to Logging API + */ + public void setLogbackBatchingSettings(LogbackBatchingSettings batchingSettings) { + logbackBatchingSettings = batchingSettings; + } + /** * Sets the flag indicating if a batch's valid entries should be written even if some other entry * failed due to an error. @@ -430,6 +458,8 @@ protected LoggingOptions getLoggingOptions() { } // opt-out metadata auto-population to control it in the appender code builder.setAutoPopulateMetadata(false); + builder.setBatchingSettings( + this.logbackBatchingSettings != null ? this.logbackBatchingSettings.build() : null); loggingOptions = builder.build(); } return loggingOptions; diff --git a/src/test/java/com/google/cloud/logging/logback/LoggingAppenderLogbackTest.java b/src/test/java/com/google/cloud/logging/logback/LoggingAppenderLogbackTest.java new file mode 100644 index 000000000..7691549e1 --- /dev/null +++ b/src/test/java/com/google/cloud/logging/logback/LoggingAppenderLogbackTest.java @@ -0,0 +1,57 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.logging.logback; + +import static com.google.common.truth.Truth.assertThat; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import ch.qos.logback.core.joran.spi.JoranException; +import com.google.api.gax.batching.FlowController.LimitExceededBehavior; +import com.google.cloud.logging.LoggingOptions; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LoggingAppenderLogbackTest { + @Test + public void testLoggingOptionsFromLogbackXMLFileConfig() throws JoranException { + LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator jc = new JoranConfigurator(); + jc.setContext(context); + context.reset(); + jc.doConfigure("src/test/java/com/google/cloud/logging/logback/logback.xml"); + Logger logger = LoggerFactory.getLogger(LoggingAppenderLogbackTest.class); + assertThat(logger.getName()) + .isEqualTo("com.google.cloud.logging.logback.LoggingAppenderLogbackTest"); + LoggingAppender appender = (LoggingAppender) context.getLogger("ROOT").getAppender("CLOUD"); + LoggingOptions options = appender.getLoggingOptions(); + assertThat(options.getAutoPopulateMetadata()).isEqualTo(false); + assertThat(options.getBatchingSettings().getDelayThreshold().toMillis()).isEqualTo(500); + assertThat(options.getBatchingSettings().getElementCountThreshold()).isEqualTo(100); + assertThat(options.getBatchingSettings().getIsEnabled()).isEqualTo(true); + assertThat(options.getBatchingSettings().getRequestByteThreshold()).isEqualTo(1000); + assertThat(options.getBatchingSettings().getFlowControlSettings().getLimitExceededBehavior()) + .isEqualTo(LimitExceededBehavior.Ignore); + assertThat( + options.getBatchingSettings().getFlowControlSettings().getMaxOutstandingElementCount()) + .isEqualTo(10000); + assertThat( + options.getBatchingSettings().getFlowControlSettings().getMaxOutstandingRequestBytes()) + .isEqualTo(100000); + } +} diff --git a/src/test/java/com/google/cloud/logging/logback/logback.xml b/src/test/java/com/google/cloud/logging/logback/logback.xml new file mode 100644 index 000000000..f0edfc7d3 --- /dev/null +++ b/src/test/java/com/google/cloud/logging/logback/logback.xml @@ -0,0 +1,57 @@ + + + + + INFO + + + + application.log + + + WARN + + + SYNC + + + false + + + true + + + global + + + src/test/java/com/google/cloud/logging/logback/dummy-credentials.json + + + String + + + + + + true + + + + 100 + 1000 + 500 + 10000 + 100000 + Ignore + + + + + + + \ No newline at end of file From d33d6eec3511cbb7418b69d312f77c454a511dad Mon Sep 17 00:00:00 2001 From: losalex Date: Tue, 10 Jan 2023 12:46:06 -0800 Subject: [PATCH 2/8] Add space --- src/test/java/com/google/cloud/logging/logback/logback.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/google/cloud/logging/logback/logback.xml b/src/test/java/com/google/cloud/logging/logback/logback.xml index f0edfc7d3..d0ec77843 100644 --- a/src/test/java/com/google/cloud/logging/logback/logback.xml +++ b/src/test/java/com/google/cloud/logging/logback/logback.xml @@ -54,4 +54,4 @@ - \ No newline at end of file + From 8189223197b3c061f1607fc50de05cade667aaa5 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 10 Jan 2023 20:47:41 +0000 Subject: [PATCH 3/8] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 972a0bd8a..4cb8deecf 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ See [Logback filters](https://logback.qos.ch/manual/filters.html#thresholdFilter application.log - WARNING + WARN SYNC @@ -114,7 +114,22 @@ See [Logback filters](https://logback.qos.ch/manual/filters.html#thresholdFilter - true + true + + + + 100 + 1000 + 500 + 10000 + 100000 + Ignore + + From ddfcf639c9d2bdb6d8f9f94b4f5816a52c52a30b Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 10 Jan 2023 20:47:45 +0000 Subject: [PATCH 4/8] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 972a0bd8a..4cb8deecf 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ See [Logback filters](https://logback.qos.ch/manual/filters.html#thresholdFilter application.log - WARNING + WARN SYNC @@ -114,7 +114,22 @@ See [Logback filters](https://logback.qos.ch/manual/filters.html#thresholdFilter - true + true + + + + 100 + 1000 + 500 + 10000 + 100000 + Ignore + + From e61565fcb41a18e9f6b4ea8c7347ee822280d4d9 Mon Sep 17 00:00:00 2001 From: losalex Date: Tue, 10 Jan 2023 12:52:45 -0800 Subject: [PATCH 5/8] Remove redundant HTML appender tag --- .readme-partials.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.readme-partials.yaml b/.readme-partials.yaml index 4135679b4..209e8a391 100644 --- a/.readme-partials.yaml +++ b/.readme-partials.yaml @@ -57,8 +57,7 @@ custom_content: | 10000 100000 Ignore - - + From ae5166a89f8f1fb9d05b70aedff3e554055e92a3 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 10 Jan 2023 20:56:11 +0000 Subject: [PATCH 6/8] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 4cb8deecf..b959c7cd2 100644 --- a/README.md +++ b/README.md @@ -128,8 +128,7 @@ See [Logback filters](https://logback.qos.ch/manual/filters.html#thresholdFilter 10000 100000 Ignore - - + From 695039fca576371bd8357f21489ad9ab6c0cfeef Mon Sep 17 00:00:00 2001 From: losalex Date: Tue, 10 Jan 2023 17:40:00 -0800 Subject: [PATCH 7/8] Add failing dependencies --- pom.xml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 24926249a..999194011 100644 --- a/pom.xml +++ b/pom.xml @@ -137,7 +137,15 @@ com.google.protobuf protobuf-java - + + + com.google.api + gax + + + org.threeten + threetenbp + From 96103875364098274b9df2060cad101572ac16c2 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Wed, 11 Jan 2023 01:42:00 +0000 Subject: [PATCH 8/8] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b959c7cd2..5aabe0ea6 100644 --- a/README.md +++ b/README.md @@ -22,20 +22,20 @@ If you are using Maven, add this to your pom.xml file: com.google.cloud google-cloud-logging-logback - 0.129.9-alpha + 0.129.10-alpha ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-logging-logback:0.129.9-alpha' +implementation 'com.google.cloud:google-cloud-logging-logback:0.129.10-alpha' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-logging-logback" % "0.129.9-alpha" +libraryDependencies += "com.google.cloud" % "google-cloud-logging-logback" % "0.129.10-alpha" ``` ## Authentication