From 570d62201740903825b2c3778ccb98e44d511442 Mon Sep 17 00:00:00 2001 From: Simon Schneider <10846939+raynigon@users.noreply.github.com> Date: Tue, 12 Nov 2024 15:28:27 +0100 Subject: [PATCH 1/2] Fix Tee Filter --- ecs-logging-access/build.gradle | 1 + .../server/AccessLogFilterConfiguration.java | 5 +- .../access/server/CustomTeeFilter.java | 115 ++++++++++++++++++ .../access/server/TeeHttpServletRequest.java | 64 ++++++++++ .../access/server/TeeHttpServletResponse.java | 63 ++++++++++ .../access/server/TeeServletInputStream.java | 67 ++++++++++ .../access/server/TeeServletOutputStream.java | 76 ++++++++++++ ecs-logging-okhttp3/build.gradle | 11 -- .../okhttp3/EcsTransactionIdInterceptor.java | 33 ----- .../EcsTransactionIdInterceptorSpec.groovy | 67 ---------- settings.gradle | 1 - 11 files changed, 388 insertions(+), 115 deletions(-) create mode 100644 ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/CustomTeeFilter.java create mode 100644 ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/TeeHttpServletRequest.java create mode 100644 ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/TeeHttpServletResponse.java create mode 100644 ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/TeeServletInputStream.java create mode 100644 ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/TeeServletOutputStream.java delete mode 100644 ecs-logging-okhttp3/build.gradle delete mode 100644 ecs-logging-okhttp3/src/main/java/com/raynigon/ecs/logging/okhttp3/EcsTransactionIdInterceptor.java delete mode 100644 ecs-logging-okhttp3/src/test/groovy/com/raynigon/ecs/logging/okhttp3/EcsTransactionIdInterceptorSpec.groovy diff --git a/ecs-logging-access/build.gradle b/ecs-logging-access/build.gradle index 55a9402..730e1f4 100644 --- a/ecs-logging-access/build.gradle +++ b/ecs-logging-access/build.gradle @@ -4,6 +4,7 @@ dependencies { implementation("ch.qos.logback.access:tomcat:2.0.3"){ exclude group: 'org.apache.tomcat' } + implementation('org.apache.commons:commons-io:1.3.2') compileOnly("org.springframework.boot:spring-boot-starter-web") compileOnly("org.springframework.boot:spring-boot-starter-webflux") diff --git a/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/AccessLogFilterConfiguration.java b/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/AccessLogFilterConfiguration.java index 19cdddb..1d222f5 100644 --- a/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/AccessLogFilterConfiguration.java +++ b/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/AccessLogFilterConfiguration.java @@ -1,6 +1,5 @@ package com.raynigon.ecs.logging.access.server; -import ch.qos.logback.access.common.servlet.TeeFilter; import com.raynigon.ecs.logging.access.AccessLogProperties; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.AutoConfigureBefore; @@ -28,7 +27,7 @@ public class AccessLogFilterConfiguration { @Bean @NonNull @ConditionalOnProperty(value = "raynigon.logging.access.export-body", havingValue = "true") - public FilterRegistrationBean requestLoggingFilter() { - return new FilterRegistrationBean<>(new TeeFilter()); + public FilterRegistrationBean requestLoggingFilter() { + return new FilterRegistrationBean<>(new CustomTeeFilter()); } } diff --git a/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/CustomTeeFilter.java b/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/CustomTeeFilter.java new file mode 100644 index 0000000..a56461e --- /dev/null +++ b/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/CustomTeeFilter.java @@ -0,0 +1,115 @@ +package com.raynigon.ecs.logging.access.server; + +import ch.qos.logback.access.common.AccessConstants; +import jakarta.servlet.*; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; + +public class CustomTeeFilter implements Filter { + boolean active; + + @Override + public void destroy() { + // NOP + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain filterChain) + throws IOException, ServletException { + + if (active && request instanceof HttpServletRequest) { + try { + TeeHttpServletRequest teeRequest = new TeeHttpServletRequest((HttpServletRequest) request); + TeeHttpServletResponse teeResponse = new TeeHttpServletResponse((HttpServletResponse) response); + + // System.out.println("BEFORE TeeFilter. filterChain.doFilter()"); + filterChain.doFilter(teeRequest, teeResponse); + // System.out.println("AFTER TeeFilter. filterChain.doFilter()"); + + teeResponse.finish(); + // let the output contents be available for later use by + // logback-access-logging + teeRequest.setAttribute(AccessConstants.LB_OUTPUT_BUFFER, teeResponse.getOutputBuffer()); + } catch (IOException e) { + e.printStackTrace(); + throw e; + } catch (ServletException e) { + e.printStackTrace(); + throw e; + } + } else { + filterChain.doFilter(request, response); + } + + } + + @Override + public void init(FilterConfig filterConfig) throws ServletException { + String includeListAsStr = filterConfig.getInitParameter(AccessConstants.TEE_FILTER_INCLUDES_PARAM); + String excludeListAsStr = filterConfig.getInitParameter(AccessConstants.TEE_FILTER_EXCLUDES_PARAM); + String localhostName = getLocalhostName(); + + active = computeActivation(localhostName, includeListAsStr, excludeListAsStr); + if (active) + System.out.println("TeeFilter will be ACTIVE on this host [" + localhostName + "]"); + else + System.out.println("TeeFilter will be DISABLED on this host [" + localhostName + "]"); + + } + + public static List extractNameList(String nameListAsStr) { + List nameList = new ArrayList(); + if (nameListAsStr == null) { + return nameList; + } + + nameListAsStr = nameListAsStr.trim(); + if (nameListAsStr.length() == 0) { + return nameList; + } + + String[] nameArray = nameListAsStr.split("[,;]"); + for (String n : nameArray) { + n = n.trim(); + nameList.add(n); + } + return nameList; + } + + static String getLocalhostName() { + String hostname = "127.0.0.1"; + + try { + hostname = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException uhe) { + uhe.printStackTrace(); + } + return hostname; + } + + public static boolean computeActivation(String hostname, String includeListAsStr, String excludeListAsStr) { + List includeList = extractNameList(includeListAsStr); + List excludeList = extractNameList(excludeListAsStr); + boolean inIncludesList = mathesIncludesList(hostname, includeList); + boolean inExcludesList = mathesExcludesList(hostname, excludeList); + return inIncludesList && (!inExcludesList); + } + + static boolean mathesIncludesList(String hostname, List includeList) { + if (includeList.isEmpty()) + return true; + return includeList.contains(hostname); + } + + static boolean mathesExcludesList(String hostname, List excludesList) { + if (excludesList.isEmpty()) + return false; + return excludesList.contains(hostname); + } +} diff --git a/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/TeeHttpServletRequest.java b/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/TeeHttpServletRequest.java new file mode 100644 index 0000000..5af4768 --- /dev/null +++ b/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/TeeHttpServletRequest.java @@ -0,0 +1,64 @@ +package com.raynigon.ecs.logging.access.server; + +import ch.qos.logback.access.common.AccessConstants; +import ch.qos.logback.access.common.servlet.Util; +import jakarta.servlet.ServletInputStream; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletRequestWrapper; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; + +public class TeeHttpServletRequest extends HttpServletRequestWrapper { + + private TeeServletInputStream inStream; + private BufferedReader reader; + boolean postedParametersMode = false; + + TeeHttpServletRequest(HttpServletRequest request) { + super(request); + // we can't access the input stream and access the request parameters + // at the same time + if (Util.isFormUrlEncoded(request)) { + postedParametersMode = true; + } else { + inStream = new TeeServletInputStream(request); + // add the contents of the input buffer as an attribute of the request + request.setAttribute(AccessConstants.LB_INPUT_BUFFER, inStream.getInputBuffer()); + reader = new BufferedReader(new InputStreamReader(inStream)); + } + + } + + byte[] getInputBuffer() { + if (postedParametersMode) { + throw new IllegalStateException("Call disallowed in postedParametersMode"); + } + return inStream.getInputBuffer(); + } + + @Override + public ServletInputStream getInputStream() throws IOException { + if (!postedParametersMode) { + return inStream; + } else { + return super.getInputStream(); + } + } + + // + + @Override + public BufferedReader getReader() throws IOException { + if (!postedParametersMode) { + return reader; + } else { + return super.getReader(); + } + } + + public boolean isPostedParametersMode() { + return postedParametersMode; + } +} diff --git a/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/TeeHttpServletResponse.java b/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/TeeHttpServletResponse.java new file mode 100644 index 0000000..8611811 --- /dev/null +++ b/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/TeeHttpServletResponse.java @@ -0,0 +1,63 @@ +package com.raynigon.ecs.logging.access.server; + + +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.servlet.http.HttpServletResponseWrapper; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; + +public class TeeHttpServletResponse extends HttpServletResponseWrapper { + + TeeServletOutputStream teeServletOutputStream; + PrintWriter teeWriter; + + public TeeHttpServletResponse(HttpServletResponse httpServletResponse) { + super(httpServletResponse); + } + + @Override + public ServletOutputStream getOutputStream() throws IOException { + if (teeServletOutputStream == null) { + teeServletOutputStream = new TeeServletOutputStream(this.getResponse()); + } + return teeServletOutputStream; + } + + @Override + public PrintWriter getWriter() throws IOException { + if (this.teeWriter == null) { + this.teeWriter = new PrintWriter( + new OutputStreamWriter(getOutputStream(), this.getResponse().getCharacterEncoding()), true); + } + return this.teeWriter; + } + + @Override + public void flushBuffer() { + if (this.teeWriter != null) { + this.teeWriter.flush(); + } + } + + public byte[] getOutputBuffer() { + // teeServletOutputStream can be null if the getOutputStream method is never + // called. + if (teeServletOutputStream != null) { + return teeServletOutputStream.getOutputStreamAsByteArray(); + } else { + return null; + } + } + + void finish() throws IOException { + if (this.teeWriter != null) { + this.teeWriter.close(); + } + if (this.teeServletOutputStream != null) { + this.teeServletOutputStream.close(); + } + } +} \ No newline at end of file diff --git a/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/TeeServletInputStream.java b/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/TeeServletInputStream.java new file mode 100644 index 0000000..9683c92 --- /dev/null +++ b/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/TeeServletInputStream.java @@ -0,0 +1,67 @@ +package com.raynigon.ecs.logging.access.server; + +import jakarta.servlet.ReadListener; +import jakarta.servlet.ServletInputStream; +import jakarta.servlet.http.HttpServletRequest; +import org.apache.commons.io.IOUtils; + +import java.io.ByteArrayInputStream; +import java.io.IOException; + +public class TeeServletInputStream extends ServletInputStream { + + private byte[] content; + private ByteArrayInputStream in; + + TeeServletInputStream(HttpServletRequest request) { + duplicateInputStream(request); + } + + private void duplicateInputStream(HttpServletRequest request) { + ServletInputStream originalSIS = null; + try { + originalSIS = request.getInputStream(); + content = IOUtils.toByteArray(request.getInputStream()); + in = new ByteArrayInputStream(content); + } catch (IOException e) { + e.printStackTrace(); + } finally { + closeStream(originalSIS); + } + } + + @Override + public int read() throws IOException { + return in.read(); + } + + void closeStream(ServletInputStream is) { + if (is != null) { + try { + is.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + byte[] getInputBuffer() { + return content; + } + + @Override + public boolean isFinished() { + throw new RuntimeException("Not yet implemented"); + } + + @Override + public boolean isReady() { + throw new RuntimeException("Not yet implemented"); + } + + @Override + public void setReadListener(ReadListener listener) { + throw new RuntimeException("Not yet implemented"); + } +} + diff --git a/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/TeeServletOutputStream.java b/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/TeeServletOutputStream.java new file mode 100644 index 0000000..19f7963 --- /dev/null +++ b/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/TeeServletOutputStream.java @@ -0,0 +1,76 @@ +package com.raynigon.ecs.logging.access.server; + + +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.WriteListener; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +public class TeeServletOutputStream extends ServletOutputStream { + + final ServletOutputStream underlyingStream; + final ByteArrayOutputStream baosCopy; + + TeeServletOutputStream(ServletResponse httpServletResponse) throws IOException { + this.underlyingStream = httpServletResponse.getOutputStream(); + baosCopy = new ByteArrayOutputStream(); + } + + byte[] getOutputStreamAsByteArray() { + return baosCopy.toByteArray(); + } + + @Override + public void write(int val) throws IOException { + if (underlyingStream != null) { + underlyingStream.write(val); + baosCopy.write(val); + } + } + + @Override + public void write(byte[] byteArray) throws IOException { + if (underlyingStream == null) { + return; + } + write(byteArray, 0, byteArray.length); + } + + @Override + public void write(byte byteArray[], int offset, int length) throws IOException { + if (underlyingStream == null) { + return; + } + underlyingStream.write(byteArray, offset, length); + baosCopy.write(byteArray, offset, length); + } + + @Override + public void close() throws IOException { + // If the servlet accessing the stream is using a writer instead of + // an OutputStream, it will probably call os.close() before calling + // writer.close. Thus, the underlying output stream will be called + // before the data sent to the writer could be flushed. + } + + @Override + public void flush() throws IOException { + if (underlyingStream == null) { + return; + } + underlyingStream.flush(); + baosCopy.flush(); + } + + @Override + public boolean isReady() { + throw new RuntimeException("Not yet implemented"); + } + + @Override + public void setWriteListener(WriteListener listener) { + throw new RuntimeException("Not yet implemented"); + } +} diff --git a/ecs-logging-okhttp3/build.gradle b/ecs-logging-okhttp3/build.gradle deleted file mode 100644 index f1948ab..0000000 --- a/ecs-logging-okhttp3/build.gradle +++ /dev/null @@ -1,11 +0,0 @@ -dependencies { - implementation(project(':ecs-logging-base')) - implementation("org.slf4j:slf4j-api:2.0.16") - compileOnly("com.squareup.okhttp3:okhttp:4.12.0") - compileOnly("ch.qos.logback:logback-core") - - testImplementation("ch.qos.logback:logback-core") - testImplementation("ch.qos.logback:logback-classic") - testImplementation("com.squareup.okhttp3:okhttp:4.12.0") - testImplementation("org.springframework.boot:spring-boot-starter-test") -} diff --git a/ecs-logging-okhttp3/src/main/java/com/raynigon/ecs/logging/okhttp3/EcsTransactionIdInterceptor.java b/ecs-logging-okhttp3/src/main/java/com/raynigon/ecs/logging/okhttp3/EcsTransactionIdInterceptor.java deleted file mode 100644 index 06f3b5b..0000000 --- a/ecs-logging-okhttp3/src/main/java/com/raynigon/ecs/logging/okhttp3/EcsTransactionIdInterceptor.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.raynigon.ecs.logging.okhttp3; - -import okhttp3.Interceptor; -import okhttp3.Request; -import okhttp3.Response; -import org.jetbrains.annotations.NotNull; -import org.slf4j.MDC; - -import java.io.IOException; -import java.util.UUID; - -import static com.raynigon.ecs.logging.LoggingConstants.TRANSACTION_ID_HEADER; -import static com.raynigon.ecs.logging.LoggingConstants.TRANSACTION_ID_PROPERTY; - -public class EcsTransactionIdInterceptor implements Interceptor { - - @NotNull - @Override - public Response intercept(Interceptor.Chain chain) throws IOException { - Request original = chain.request(); - String transactionId = MDC.get(TRANSACTION_ID_PROPERTY); - // If no transaction id exists, generate custom transaction id - if (transactionId == null) { - transactionId = UUID.randomUUID().toString(); - MDC.put(TRANSACTION_ID_PROPERTY, transactionId); - } - // Update Request with transaction id header - Request updated = original.newBuilder() - .header(TRANSACTION_ID_HEADER, transactionId) - .build(); - return chain.proceed(updated); - } -} \ No newline at end of file diff --git a/ecs-logging-okhttp3/src/test/groovy/com/raynigon/ecs/logging/okhttp3/EcsTransactionIdInterceptorSpec.groovy b/ecs-logging-okhttp3/src/test/groovy/com/raynigon/ecs/logging/okhttp3/EcsTransactionIdInterceptorSpec.groovy deleted file mode 100644 index 085ae40..0000000 --- a/ecs-logging-okhttp3/src/test/groovy/com/raynigon/ecs/logging/okhttp3/EcsTransactionIdInterceptorSpec.groovy +++ /dev/null @@ -1,67 +0,0 @@ -package com.raynigon.ecs.logging.okhttp3 - -import static com.raynigon.ecs.logging.LoggingConstants.TRANSACTION_ID_HEADER -import static com.raynigon.ecs.logging.LoggingConstants.TRANSACTION_ID_PROPERTY - -import okhttp3.Protocol -import okhttp3.Request -import okhttp3.Response -import org.slf4j.MDC -import spock.lang.Specification -import spock.lang.Subject -import okhttp3.Interceptor - -class EcsTransactionIdInterceptorSpec extends Specification { - - @Subject - EcsTransactionIdInterceptor interceptor = new EcsTransactionIdInterceptor() - - def "transaction id header exists"() { - given: - Interceptor.Chain chain = Mock() - Request request = new Request.Builder().url("https://example.com").build() - Response response = new Response.Builder().request(request).code(200) - .protocol(Protocol.HTTP_1_1) - .message("dummy") - .build() - - when: - def result = interceptor.intercept(chain) - - then: - 1 * chain.request() >> request - 1 * chain.proceed({ Request r -> r.header(TRANSACTION_ID_HEADER) != null }) >> response - - and: - result == response - - cleanup: - MDC.clear() - } - - def "transaction id header is set from MDC Tag"() { - given: - Interceptor.Chain chain = Mock() - Request request = new Request.Builder().url("https://example.com").build() - Response response = new Response.Builder().request(request).code(200) - .protocol(Protocol.HTTP_1_1) - .message("dummy") - .build() - - and: - MDC.put(TRANSACTION_ID_PROPERTY, "my-value") - - when: - def result = interceptor.intercept(chain) - - then: - 1 * chain.request() >> request - 1 * chain.proceed({ Request r -> r.header(TRANSACTION_ID_HEADER) == "my-value" }) >> response - - and: - result == response - - cleanup: - MDC.clear() - } -} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 299611d..67ed8f5 100644 --- a/settings.gradle +++ b/settings.gradle @@ -6,5 +6,4 @@ include 'ecs-logging-access' include 'ecs-logging-async' include 'ecs-logging-audit' include 'ecs-logging-kafka' -include 'ecs-logging-okhttp3' include 'gzip-request-filter-starter' \ No newline at end of file From d18f67ef1ef1480c7fc43124a1cf405e25ef5cef Mon Sep 17 00:00:00 2001 From: Simon Schneider <10846939+raynigon@users.noreply.github.com> Date: Wed, 13 Nov 2024 10:07:13 +0100 Subject: [PATCH 2/2] Fix Tee Filter --- .../logging/access/server/CustomTeeFilter.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/CustomTeeFilter.java b/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/CustomTeeFilter.java index a56461e..4df4de8 100644 --- a/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/CustomTeeFilter.java +++ b/ecs-logging-access/src/main/java/com/raynigon/ecs/logging/access/server/CustomTeeFilter.java @@ -28,9 +28,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha TeeHttpServletRequest teeRequest = new TeeHttpServletRequest((HttpServletRequest) request); TeeHttpServletResponse teeResponse = new TeeHttpServletResponse((HttpServletResponse) response); - // System.out.println("BEFORE TeeFilter. filterChain.doFilter()"); filterChain.doFilter(teeRequest, teeResponse); - // System.out.println("AFTER TeeFilter. filterChain.doFilter()"); teeResponse.finish(); // let the output contents be available for later use by @@ -56,11 +54,12 @@ public void init(FilterConfig filterConfig) throws ServletException { String localhostName = getLocalhostName(); active = computeActivation(localhostName, includeListAsStr, excludeListAsStr); - if (active) + // Log the activation status on stdout, because loggers cannot be used here + if (active) { System.out.println("TeeFilter will be ACTIVE on this host [" + localhostName + "]"); - else + } else { System.out.println("TeeFilter will be DISABLED on this host [" + localhostName + "]"); - + } } public static List extractNameList(String nameListAsStr) { @@ -96,18 +95,18 @@ static String getLocalhostName() { public static boolean computeActivation(String hostname, String includeListAsStr, String excludeListAsStr) { List includeList = extractNameList(includeListAsStr); List excludeList = extractNameList(excludeListAsStr); - boolean inIncludesList = mathesIncludesList(hostname, includeList); - boolean inExcludesList = mathesExcludesList(hostname, excludeList); + boolean inIncludesList = matchesIncludesList(hostname, includeList); + boolean inExcludesList = matchesExcludesList(hostname, excludeList); return inIncludesList && (!inExcludesList); } - static boolean mathesIncludesList(String hostname, List includeList) { + static boolean matchesIncludesList(String hostname, List includeList) { if (includeList.isEmpty()) return true; return includeList.contains(hostname); } - static boolean mathesExcludesList(String hostname, List excludesList) { + static boolean matchesExcludesList(String hostname, List excludesList) { if (excludesList.isEmpty()) return false; return excludesList.contains(hostname);